Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
38.2 KB
Raw Blame
/**
 * v0.41.22.2 — pure-function unit tests for `runLockRenewalTick`.
 *
 * Pins all 14 state-machine paths the cathedral cares about:
 *
 *   - happy-path single renewal
 *   - throw bookkeeping (counter, audit, no abort if within deadline)
 *   - recovery (success_after_failure audit, counter reset)
 *   - time-based abort (gave_up audit, abort returned)
 *   - lock_lost (token mismatch, no audit, no infrastructure framing)
 *   - hung renewLock (Promise.race timeout fires, counter increments)
 *   - cancellation at three points (entry, after-resolve, after-reject)
 *   - time-based-vs-count-based: deadline crosses BEFORE the counter
 *     hits `maxFailuresForAudit`, abort still fires
 *   - audit-throw defense-in-depth (audit threw, tick still returns
 *     `'ok'` — the headline regression for the v0.41.22.1 bug class
 *     this whole wave exists to close)
 *   - env-knob resolution (good values, bad values fallback +
 *     stderr-warn-once)
 *
 * Hermetic: no fs, no PGLite, no real Anthropic, no setInterval. The
 * tick function takes ALL effects via the injected `deps` object so
 * everything is a vanilla async-function call.
 */

import { describe, expect, test, beforeEach } from 'bun:test';
import {
  runLockRenewalTick,
  resolveLockRenewalKnobs,
  _resetKnobWarningsForTests,
  type LockRenewalDeps,
  type LockRenewalState,
  type LockRenewalAuditSinkLike,
  type LockRenewalKnobs,
} from '../src/core/minions/lock-renewal-tick.ts';
import { withEnv } from './helpers/with-env.ts';

// --- fakes ----------------------------------------------------------------

interface AuditLog {
  failures: Array<{ jobId: number; jobName: string; attempt: number; err: unknown; ctx?: unknown }>;
  recoveries: Array<{ jobId: number; jobName: string; recoveredAfterAttempts: number; ctx?: unknown }>;
  gaveUps: Array<{ jobId: number; jobName: string; totalFailures: number; err: unknown; ctx?: unknown }>;
}

function freshAudit(): { sink: LockRenewalAuditSinkLike; log: AuditLog } {
  const log: AuditLog = { failures: [], recoveries: [], gaveUps: [] };
  return {
    log,
    sink: {
      logFailure: (jobId, jobName, attempt, err, ctx) =>
        log.failures.push({ jobId, jobName, attempt, err, ctx }),
      logSuccessAfterFailure: (jobId, jobName, recoveredAfterAttempts, ctx) =>
        log.recoveries.push({ jobId, jobName, recoveredAfterAttempts, ctx }),
      logGaveUp: (jobId, jobName, totalFailures, err, ctx) =>
        log.gaveUps.push({ jobId, jobName, totalFailures, err, ctx }),
    },
  };
}

/**
 * Fake setTimeout that records the requested ms but does NOT actually
 * schedule a real timer (no leaks across tests). The callback never
 * fires unless tests explicitly call `runTimers(times)`.
 */
function makeFakeTimer(): {
  setTimeout: LockRenewalDeps['setTimeout'];
  runAll: () => void;
  pending: Array<() => void>;
} {
  const pending: Array<() => void> = [];
  return {
    setTimeout: (cb) => {
      pending.push(cb);
      return null;
    },
    runAll: () => {
      const toRun = [...pending];
      pending.length = 0;
      toRun.forEach((cb) => cb());
    },
    pending,
  };
}

const DEFAULT_LOCK_MS = 30_000;
const DEFAULT_KNOBS: LockRenewalKnobs = {
  maxFailuresForAudit: 3,
  callTimeoutMs: 10_000,
  safetyMarginMs: 5_000,
  hardEvictMs: 60_000, // 2 × lockDuration (the #4145 backstop)
};

function makeState(overrides?: Partial<LockRenewalState>): LockRenewalState {
  return {
    jobId: 42,
    jobName: 'sync',
    lockToken: 'tok-abc',
    lockDurationMs: DEFAULT_LOCK_MS,
    knobs: DEFAULT_KNOBS,
    lastSuccessfulRenewalAt: 0,
    consecutiveFailures: 0,
    cancelled: () => false,
    // v0.46 (#4145) telemetry fields: cadence for lateness math, seeded
    // previous-tick timestamp, and the worker-maintained overlap counter.
    intervalMs: DEFAULT_LOCK_MS / 2,
    lastTickFiredAt: 0,
    overlapSkips: 0,
    ...overrides,
  };
}

// --- tests ----------------------------------------------------------------

describe('runLockRenewalTick: happy path', () => {
  test('case 1 — first-try success returns ok, no audit, lastSuccessfulRenewalAt updated', async () => {
    const audit = freshAudit();
    const timer = makeFakeTimer();
    const deps: LockRenewalDeps = {
      renewLock: async () => true,
      audit: audit.sink,
      now: () => 1000,
      setTimeout: timer.setTimeout,
    };
    const state = makeState({ lastSuccessfulRenewalAt: 500 });
    const result = await runLockRenewalTick(deps, state);
    expect(result).toEqual({ kind: 'ok' });
    expect(audit.log.failures).toHaveLength(0);
    expect(audit.log.recoveries).toHaveLength(0);
    expect(audit.log.gaveUps).toHaveLength(0);
    expect(state.lastSuccessfulRenewalAt).toBe(1000);
    expect(state.consecutiveFailures).toBe(0);
  });
});

describe('runLockRenewalTick: failure counter + audit', () => {
  test('case 2 — single throw within deadline: returns ok, counter=1, failure logged', async () => {
    const audit = freshAudit();
    const deps: LockRenewalDeps = {
      renewLock: async () => { throw new Error('Connection terminated'); },
      audit: audit.sink,
      now: () => 1000, // sinceLastSuccess = 1000ms, deadline = 25000ms; well within
      setTimeout: makeFakeTimer().setTimeout,
    };
    const state = makeState({ lastSuccessfulRenewalAt: 0 });
    const result = await runLockRenewalTick(deps, state);
    expect(result).toEqual({ kind: 'ok' });
    expect(state.consecutiveFailures).toBe(1);
    expect(audit.log.failures).toHaveLength(1);
    expect(audit.log.failures[0].attempt).toBe(1);
    expect(audit.log.gaveUps).toHaveLength(0);
  });

  test('case 3 — two throws then success: counter resets, success_after_failure logged', async () => {
    const audit = freshAudit();
    let callIdx = 0;
    const deps: LockRenewalDeps = {
      renewLock: async () => {
        callIdx++;
        if (callIdx <= 2) throw new Error('blip ' + callIdx);
        return true;
      },
      audit: audit.sink,
      now: () => 1000,
      setTimeout: makeFakeTimer().setTimeout,
    };
    const state = makeState({ lastSuccessfulRenewalAt: 0 });

    const r1 = await runLockRenewalTick(deps, state);
    expect(r1.kind).toBe('ok');
    expect(state.consecutiveFailures).toBe(1);

    const r2 = await runLockRenewalTick(deps, state);
    expect(r2.kind).toBe('ok');
    expect(state.consecutiveFailures).toBe(2);

    const r3 = await runLockRenewalTick(deps, state);
    expect(r3.kind).toBe('ok');
    expect(state.consecutiveFailures).toBe(0);

    expect(audit.log.failures).toHaveLength(2);
    expect(audit.log.recoveries).toHaveLength(1);
    expect(audit.log.recoveries[0].recoveredAfterAttempts).toBe(2);
  });
});

describe('runLockRenewalTick: verify-before-evict + hard backstop (#4145)', () => {
  test('case 4 — sustained throws past the SOFT deadline are DEFERRED (verify attempted), abort only past hardEvictMs', async () => {
    const audit = freshAudit();
    // Soft deadline = 30000 - 5000 = 25000; hardEvict = 60000.
    // At 26s of failure the pre-#4145 code aborted — the incident. Now the
    // tick attempts a verify (which also throws here) and DEFERS.
    const deps: LockRenewalDeps = {
      renewLock: async () => { throw new Error('persistent outage'); },
      audit: audit.sink,
      now: () => 26_000,
      setTimeout: makeFakeTimer().setTimeout,
    };
    const state = makeState({ lastSuccessfulRenewalAt: 0, consecutiveFailures: 2 });
    const deferred = await runLockRenewalTick(deps, state);
    expect(deferred).toEqual({ kind: 'ok' });
    // Primary attempt (3) + failed verify (4), the latter deadline_deferred.
    expect(state.consecutiveFailures).toBe(4);
    expect(audit.log.failures).toHaveLength(2);
    expect(audit.log.failures[1].ctx).toMatchObject({ deadline_deferred: true, cause: 'refused' });
    expect(audit.log.gaveUps).toHaveLength(0);

    // Past the hard backstop the tick gives up: primary (5) + verify (6),
    // gave_up carries the classified cause + starvation telemetry.
    // lateness = max(0, 61000 - 26000 - 15000) = 20000.
    const hardDeps: LockRenewalDeps = { ...deps, now: () => 61_000 };
    const result = await runLockRenewalTick(hardDeps, state);
    expect(result).toMatchObject({
      kind: 'should_abort',
      reason: 'lock-renewal-failed',
      cause: 'refused',
      sinceLastSuccessMs: 61_000,
      latenessMs: 20_000,
      overlapSkips: 0,
    });
    expect(state.consecutiveFailures).toBe(6);
    expect(audit.log.gaveUps).toHaveLength(1);
    expect(audit.log.gaveUps[0].totalFailures).toBe(6);
  });

  test('case 10 — INCIDENT REPLAY: renewal times out under starvation, verify succeeds → job survives', async () => {
    // The #4145 incident shape: the event loop starves, the renewal's
    // 10s race timeout wins even though the DB is healthy (external
    // SELECT 1 probes ran 32ms). Pre-fix: abort at the 25s deadline,
    // discarding ~173s of LLM work. Post-fix: ONE fenced verify — the
    // DB is the authority — recovers the lease and the job survives.
    const audit = freshAudit();
    let calls = 0;
    const timer = makeFakeTimer();
    const deps: LockRenewalDeps = {
      renewLock: async () => {
        calls++;
        if (calls === 1) return new Promise<boolean>(() => { /* starved: never settles */ });
        return true; // the verify — DB was healthy all along
      },
      audit: audit.sink,
      now: () => 26_000, // past the soft deadline of 25000
      setTimeout: timer.setTimeout,
    };
    const state = makeState({ lastSuccessfulRenewalAt: 0 });
    const pending = runLockRenewalTick(deps, state);
    timer.runAll(); // fire the primary race timeout (the starvation symptom)
    const result = await pending;
    expect(result).toEqual({ kind: 'ok' }); // NO abort — the incident fix
    expect(state.consecutiveFailures).toBe(0); // verify-success resets
    expect(state.lastSuccessfulRenewalAt).toBe(26_000);
    expect(audit.log.failures).toHaveLength(1); // the primary call-timeout
    expect(audit.log.failures[0].ctx).toMatchObject({ cause: 'call-timeout' });
    expect(audit.log.recoveries).toHaveLength(1);
    expect(audit.log.recoveries[0].ctx).toMatchObject({ via: 'verify' });
    expect(audit.log.gaveUps).toHaveLength(0);
  });

  test('case 10b — verify returns fenced-false: CERTAIN loss → lock_lost via verify', async () => {
    const audit = freshAudit();
    let calls = 0;
    const deps: LockRenewalDeps = {
      renewLock: async () => {
        calls++;
        if (calls === 1) throw new Error('outage');
        return false; // the stall sweep reclaimed the row during the blip
      },
      audit: audit.sink,
      now: () => 26_000,
      setTimeout: makeFakeTimer().setTimeout,
    };
    const state = makeState({ lastSuccessfulRenewalAt: 0 });
    const result = await runLockRenewalTick(deps, state);
    expect(result).toEqual({ kind: 'lock_lost', cause: 'fenced-lost', via: 'verify' });
    expect(audit.log.gaveUps).toHaveLength(0); // certain loss ≠ infrastructure gave-up
  });

  test('case 10c — CDX-4 cadence quantization: 300s lease / 60s cadence verifies at 240s with lease remaining', async () => {
    // With the bare `>= deadline` gate there is NO tick between 240s and
    // lease expiry (300s): deadline 270s is unreachable and the first
    // eligible tick is already past expiry. The next-tick-too-late form
    // (sinceLastSuccess + intervalMs >= deadline) verifies at 240s.
    const audit = freshAudit();
    const knobs: LockRenewalKnobs = {
      maxFailuresForAudit: 3,
      callTimeoutMs: 15_000,
      safetyMarginMs: 30_000, // deadline = 270_000
      hardEvictMs: 600_000,
    };
    let calls = 0;
    const deps: LockRenewalDeps = {
      renewLock: async () => {
        calls++;
        if (calls === 1) throw new Error('blip at the 240s tick');
        return true; // verify recovers with 60s of lease left
      },
      audit: audit.sink,
      now: () => 240_000,
      setTimeout: makeFakeTimer().setTimeout,
    };
    const state = makeState({
      lockDurationMs: 300_000,
      knobs,
      intervalMs: 60_000,
      lastSuccessfulRenewalAt: 0,
      lastTickFiredAt: 180_000,
    });
    const result = await runLockRenewalTick(deps, state);
    expect(result).toEqual({ kind: 'ok' });
    expect(calls).toBe(2); // primary + verify — the verify DID fire at 240s
    expect(audit.log.recoveries[0].ctx).toMatchObject({ via: 'verify' });

    // Control: at the 180s tick (180 + 60 = 240 < 270) no verify fires.
    calls = 0;
    const controlState = makeState({
      lockDurationMs: 300_000,
      knobs,
      intervalMs: 60_000,
      lastSuccessfulRenewalAt: 0,
      lastTickFiredAt: 120_000,
    });
    const controlDeps: LockRenewalDeps = {
      ...deps,
      renewLock: async () => { calls++; throw new Error('blip'); },
      now: () => 180_000,
    };
    expect(await runLockRenewalTick(controlDeps, controlState)).toEqual({ kind: 'ok' });
    expect(calls).toBe(1); // primary only — deferred to the next tick
  });

  test('case 10d — ENG-E1 pin: 30s default verifies on the FIRST failed tick (15s), strictly safer than abort-at-30s', async () => {
    // interval 15s, deadline 25s: at the t=15s tick, 15000 + 15000 >= 25000
    // → the verify fires immediately on the first failure. This is the
    // accepted timing behavior — a renewal(≤10s) + verify(≤10s) pair can
    // span 20s and skip one tick via tickInFlight; benign because a
    // verify-success re-extends the lease.
    let calls = 0;
    const deps: LockRenewalDeps = {
      renewLock: async () => {
        calls++;
        if (calls === 1) throw new Error('first-tick blip');
        return true;
      },
      audit: freshAudit().sink,
      now: () => 15_000,
      setTimeout: makeFakeTimer().setTimeout,
    };
    const state = makeState({ lastSuccessfulRenewalAt: 0 });
    expect(await runLockRenewalTick(deps, state)).toEqual({ kind: 'ok' });
    expect(calls).toBe(2);
    expect(state.lastSuccessfulRenewalAt).toBe(15_000);
  });

  test('case 10e — cancelled() flips mid-verify: returns cancelled, no abort', async () => {
    let cancelled = false;
    let calls = 0;
    const deps: LockRenewalDeps = {
      renewLock: async () => {
        calls++;
        if (calls === 1) throw new Error('outage');
        cancelled = true; // the job ends while the verify is in flight
        return true;
      },
      audit: freshAudit().sink,
      now: () => 26_000,
      setTimeout: makeFakeTimer().setTimeout,
    };
    const state = makeState({ lastSuccessfulRenewalAt: 0, cancelled: () => cancelled });
    const result = await runLockRenewalTick(deps, state);
    expect(result).toEqual({ kind: 'cancelled' });
  });
});

describe('runLockRenewalTick: lock_lost (token mismatch)', () => {
  test('case 5 — renewLock returns false: lock_lost, NO audit event', async () => {
    const audit = freshAudit();
    const deps: LockRenewalDeps = {
      renewLock: async () => false,
      audit: audit.sink,
      now: () => 1000,
      setTimeout: makeFakeTimer().setTimeout,
    };
    const state = makeState({ lastSuccessfulRenewalAt: 0 });
    const result = await runLockRenewalTick(deps, state);
    // v0.46 (#4145): lock_lost names its cause — the fenced UPDATE matched
    // 0 rows, the only CERTAIN loss signal.
    expect(result).toEqual({ kind: 'lock_lost', cause: 'fenced-lost', via: 'renewal' });
    expect(audit.log.failures).toHaveLength(0);
    expect(audit.log.gaveUps).toHaveLength(0);
    expect(audit.log.recoveries).toHaveLength(0);
  });
});

describe('runLockRenewalTick: hung renewLock timeout (codex C3)', () => {
  test('case 6 — renewLock hangs past callTimeoutMs: Promise.race fires, counter increments', async () => {
    const audit = freshAudit();
    const timer = makeFakeTimer();
    // renewLock never resolves; only the timeout race rejects.
    const deps: LockRenewalDeps = {
      renewLock: () => new Promise<boolean>(() => { /* never resolves */ }),
      audit: audit.sink,
      now: () => 1000,
      setTimeout: timer.setTimeout,
    };
    const state = makeState({ lastSuccessfulRenewalAt: 0 });

    // Start the tick. It awaits Promise.race; the renewLock promise
    // never resolves, but as soon as we fire the timeout callback, the
    // race rejects.
    const tickPromise = runLockRenewalTick(deps, state);
    // Yield once so the Promise.race actually wires up the deps.setTimeout call.
    await new Promise((r) => setImmediate(r));
    expect(timer.pending.length).toBeGreaterThan(0);
    timer.runAll();

    const result = await tickPromise;
    expect(result).toEqual({ kind: 'ok' }); // counter incremented, within deadline
    expect(state.consecutiveFailures).toBe(1);
    expect(audit.log.failures).toHaveLength(1);
    expect(audit.log.failures[0].err).toBeInstanceOf(Error);
    expect((audit.log.failures[0].err as Error).message).toMatch(/timed out after 10000ms/);
  });
});

describe('runLockRenewalTick: cancellation', () => {
  test('case 7 — cancelled BEFORE tick fires: returns cancelled immediately, no audit, no renewLock call', async () => {
    const audit = freshAudit();
    let renewLockCalled = false;
    const deps: LockRenewalDeps = {
      renewLock: async () => { renewLockCalled = true; return true; },
      audit: audit.sink,
      now: () => 1000,
      setTimeout: makeFakeTimer().setTimeout,
    };
    const state = makeState({ cancelled: () => true });
    const result = await runLockRenewalTick(deps, state);
    expect(result).toEqual({ kind: 'cancelled' });
    expect(renewLockCalled).toBe(false);
    expect(audit.log.failures).toHaveLength(0);
  });

  test('case 8 — cancelled DURING renewLock await (resolved branch): returns cancelled, no audit', async () => {
    const audit = freshAudit();
    let cancelled = false;
    let renewLockResolve: ((v: boolean) => void) | undefined;
    const deps: LockRenewalDeps = {
      renewLock: () => new Promise<boolean>((res) => { renewLockResolve = res; }),
      audit: audit.sink,
      now: () => 1000,
      setTimeout: makeFakeTimer().setTimeout,
    };
    const state = makeState({ cancelled: () => cancelled });

    const tick = runLockRenewalTick(deps, state);
    // Flip cancelled, THEN resolve the renewLock. The post-await branch
    // must check cancelled and bail.
    cancelled = true;
    renewLockResolve!(true);
    const result = await tick;
    expect(result).toEqual({ kind: 'cancelled' });
    // Even though renewLock returned true, no recovery audit fires
    // because cancelled gated it.
    expect(audit.log.recoveries).toHaveLength(0);
  });

  test('case 9 — cancelled DURING renewLock await (thrown branch): returns cancelled, no audit', async () => {
    const audit = freshAudit();
    let cancelled = false;
    let renewLockReject: ((e: Error) => void) | undefined;
    const deps: LockRenewalDeps = {
      renewLock: () => new Promise<boolean>((_, rej) => { renewLockReject = rej; }),
      audit: audit.sink,
      now: () => 1000,
      setTimeout: makeFakeTimer().setTimeout,
    };
    const state = makeState({ cancelled: () => cancelled });

    const tick = runLockRenewalTick(deps, state);
    // Flip cancelled, THEN reject. The catch-branch cancellation check
    // must skip the audit AND skip the deadline math.
    cancelled = true;
    renewLockReject!(new Error('would-have-been-logged'));
    const result = await tick;
    expect(result).toEqual({ kind: 'cancelled' });
    expect(audit.log.failures).toHaveLength(0);
    expect(audit.log.gaveUps).toHaveLength(0);
  });
});

describe('runLockRenewalTick: audit defense-in-depth (codex C4)', () => {
  test('case 11 — audit.logFailure throws: tick still returns ok, counter still increments', async () => {
    const audit: LockRenewalAuditSinkLike = {
      logFailure: () => { throw new Error('audit subsystem on fire'); },
      logSuccessAfterFailure: () => { /* noop */ },
      logGaveUp: () => { /* noop */ },
    };
    const deps: LockRenewalDeps = {
      renewLock: async () => { throw new Error('outage'); },
      audit,
      now: () => 1000,
      setTimeout: makeFakeTimer().setTimeout,
    };
    const state = makeState({ lastSuccessfulRenewalAt: 0 });
    const result = await runLockRenewalTick(deps, state);
    expect(result).toEqual({ kind: 'ok' });
    expect(state.consecutiveFailures).toBe(1);
  });

  test('case 11b — audit.logGaveUp throws: tick still returns should_abort', async () => {
    const audit: LockRenewalAuditSinkLike = {
      logFailure: () => { /* noop */ },
      logSuccessAfterFailure: () => { /* noop */ },
      logGaveUp: () => { throw new Error('audit subsystem on fire'); },
    };
    const deps: LockRenewalDeps = {
      renewLock: async () => { throw new Error('outage'); },
      audit,
      // Past hardEvictMs (60000) so the verify-throws path reaches gave_up.
      now: () => 61_000,
      setTimeout: makeFakeTimer().setTimeout,
    };
    const state = makeState({ lastSuccessfulRenewalAt: 0 });
    const result = await runLockRenewalTick(deps, state);
    expect(result).toMatchObject({ kind: 'should_abort', reason: 'lock-renewal-failed' });
  });

  test('case 11c — audit.logSuccessAfterFailure throws: tick still returns ok, counter resets', async () => {
    const audit: LockRenewalAuditSinkLike = {
      logFailure: () => { /* noop */ },
      logSuccessAfterFailure: () => { throw new Error('audit subsystem on fire'); },
      logGaveUp: () => { /* noop */ },
    };
    const deps: LockRenewalDeps = {
      renewLock: async () => true,
      audit,
      now: () => 1000,
      setTimeout: makeFakeTimer().setTimeout,
    };
    const state = makeState({ consecutiveFailures: 2, lastSuccessfulRenewalAt: 0 });
    const result = await runLockRenewalTick(deps, state);
    expect(result).toEqual({ kind: 'ok' });
    expect(state.consecutiveFailures).toBe(0);
    expect(state.lastSuccessfulRenewalAt).toBe(1000);
  });
});

describe('resolveLockRenewalKnobs', () => {
  beforeEach(() => { _resetKnobWarningsForTests(); });

  test('case 12a — defaults derive from lockDuration', () => {
    const knobs = resolveLockRenewalKnobs({}, 30_000);
    expect(knobs.maxFailuresForAudit).toBe(3);
    expect(knobs.callTimeoutMs).toBe(10_000);
    expect(knobs.safetyMarginMs).toBe(5_000);
  });

  test('case 12b — valid env values parse cleanly', () => {
    const knobs = resolveLockRenewalKnobs({
      GBRAIN_LOCK_RENEWAL_MAX_FAILURES: '5',
      GBRAIN_LOCK_RENEWAL_CALL_TIMEOUT_MS: '15000',
      GBRAIN_LOCK_RENEWAL_SAFETY_MARGIN_MS: '8000',
    }, 30_000);
    expect(knobs.maxFailuresForAudit).toBe(5);
    expect(knobs.callTimeoutMs).toBe(15_000);
    expect(knobs.safetyMarginMs).toBe(8_000);
  });

  test('case 12c — bad env (abc/-5/0/1.5) falls back to default with single stderr warn', async () => {
    const captured: string[] = [];
    const origWrite = process.stderr.write.bind(process.stderr);
    (process.stderr as { write: (chunk: string | Uint8Array) => boolean }).write = (chunk) => {
      captured.push(typeof chunk === 'string' ? chunk : Buffer.from(chunk).toString('utf8'));
      return true;
    };
    try {
      _resetKnobWarningsForTests();

      // Each bad value falls back.
      let knobs = resolveLockRenewalKnobs({ GBRAIN_LOCK_RENEWAL_MAX_FAILURES: 'abc' }, 30_000);
      expect(knobs.maxFailuresForAudit).toBe(3);

      knobs = resolveLockRenewalKnobs({ GBRAIN_LOCK_RENEWAL_MAX_FAILURES: '-5' }, 30_000);
      expect(knobs.maxFailuresForAudit).toBe(3);

      knobs = resolveLockRenewalKnobs({ GBRAIN_LOCK_RENEWAL_MAX_FAILURES: '0' }, 30_000);
      expect(knobs.maxFailuresForAudit).toBe(3);

      knobs = resolveLockRenewalKnobs({ GBRAIN_LOCK_RENEWAL_MAX_FAILURES: '1.5' }, 30_000);
      expect(knobs.maxFailuresForAudit).toBe(3);

      // Despite 4 bad invocations, only ONE stderr warn per env-name fired.
      const warnLines = captured.filter((c) => c.includes('GBRAIN_LOCK_RENEWAL_MAX_FAILURES'));
      expect(warnLines).toHaveLength(1);
      expect(warnLines[0]).toContain('not a positive integer');
      expect(warnLines[0]).toContain('falling back to default 3');
    } finally {
      process.stderr.write = origWrite;
    }
  });

  test('case 12d — different env names warn independently', async () => {
    const captured: string[] = [];
    const origWrite = process.stderr.write.bind(process.stderr);
    (process.stderr as { write: (chunk: string | Uint8Array) => boolean }).write = (chunk) => {
      captured.push(typeof chunk === 'string' ? chunk : Buffer.from(chunk).toString('utf8'));
      return true;
    };
    try {
      _resetKnobWarningsForTests();
      resolveLockRenewalKnobs({
        GBRAIN_LOCK_RENEWAL_MAX_FAILURES: 'bad1',
        GBRAIN_LOCK_RENEWAL_CALL_TIMEOUT_MS: 'bad2',
        GBRAIN_LOCK_RENEWAL_SAFETY_MARGIN_MS: 'bad3',
      }, 30_000);
      const warnLines = captured.filter((c) => c.includes('not a positive integer'));
      expect(warnLines).toHaveLength(3);
    } finally {
      process.stderr.write = origWrite;
    }
  });

  test('case 14 — maxFailuresForAudit default is 3 (audit-labeling regression)', () => {
    // Pinned as a named-constant regression: a future change here is
    // a deliberate two-line edit (default + this test).
    expect(resolveLockRenewalKnobs({}, 30_000).maxFailuresForAudit).toBe(3);
  });

  test('case 15a — hardEvictMs defaults to 2×lockDuration; env override + floor to soft deadline (#4145)', () => {
    _resetKnobWarningsForTests();
    expect(resolveLockRenewalKnobs({}, 30_000).hardEvictMs).toBe(60_000);
    expect(resolveLockRenewalKnobs({ GBRAIN_LOCK_RENEWAL_HARD_EVICT_MS: '120000' }, 30_000).hardEvictMs).toBe(120_000);
    // Below the soft deadline (25000) → warn-and-floor to the deadline.
    const floored = resolveLockRenewalKnobs({ GBRAIN_LOCK_RENEWAL_HARD_EVICT_MS: '1000' }, 30_000);
    expect(floored.hardEvictMs).toBe(25_000);
  });

  test('case 15b — CDX-10 relational validation: margin >= lease/2 and callTimeout > cadence are clamped', () => {
    _resetKnobWarningsForTests();
    // Margin 20000 >= 30000/2 → clamped back to the default (5000).
    const m = resolveLockRenewalKnobs({ GBRAIN_LOCK_RENEWAL_SAFETY_MARGIN_MS: '20000' }, 30_000);
    expect(m.safetyMarginMs).toBe(5_000);
    // Call timeout 20000 > cadence 15000 → clamped to the cadence.
    const c = resolveLockRenewalKnobs({ GBRAIN_LOCK_RENEWAL_CALL_TIMEOUT_MS: '20000' }, 30_000, 15_000);
    expect(c.callTimeoutMs).toBe(15_000);
    // A valid env override still wins untouched (env-beats-default, passes validation).
    const ok = resolveLockRenewalKnobs({ GBRAIN_LOCK_RENEWAL_CALL_TIMEOUT_MS: '12000' }, 30_000, 15_000);
    expect(ok.callTimeoutMs).toBe(12_000);
  });

  test('case 15c — per-job lease knob derivation: 300s lease gets capped 15s/30s defaults + 600s hardEvict', () => {
    _resetKnobWarningsForTests();
    const knobs = resolveLockRenewalKnobs({}, 300_000, 60_000);
    expect(knobs.hardEvictMs).toBe(600_000);
    // The lock/3 and lock/6 derivations CAP at 15s/30s for long leases —
    // a 100s call timeout would wedge tickInFlight across cadence windows.
    expect(knobs.callTimeoutMs).toBe(15_000);
    expect(knobs.safetyMarginMs).toBe(30_000);
    // 30s worker default keeps today's numbers exactly.
    const legacy = resolveLockRenewalKnobs({}, 30_000, 15_000);
    expect(legacy.callTimeoutMs).toBe(10_000);
    expect(legacy.safetyMarginMs).toBe(5_000);
  });
});

// issue #1678 (Codex #2): the bounded reconnect-once hook. NOT a withRetry on
// renewLock (that races this tick's own timeout); a single pool rebuild before
// the next tick so the next renewLock hits a live connection.
describe('runLockRenewalTick: reconnect-once dep (issue #1678)', () => {
  test('reconnect fires after a renewLock throw within deadline; result still ok', async () => {
    const audit = freshAudit();
    let reconnectCalls = 0;
    const deps: LockRenewalDeps = {
      renewLock: async () => { throw new Error('write CONNECTION_ENDED'); },
      audit: audit.sink,
      now: () => 1000, // well within deadline
      setTimeout: makeFakeTimer().setTimeout,
      reconnect: async () => { reconnectCalls++; },
    };
    const state = makeState({ lastSuccessfulRenewalAt: 0 });
    const result = await runLockRenewalTick(deps, state);
    expect(result).toEqual({ kind: 'ok' });
    expect(reconnectCalls).toBe(1);
    expect(state.consecutiveFailures).toBe(1);
  });

  // CODEX impl review #2 (#1685 GAP B): the tick must thread the triggering
  // renewLock error to reconnect, so PostgresEngine.reconnect can classify a
  // CONNECTION_ENDED pooler reap as reap_detected (not reconnect_other) for
  // pool_reap_health. Pin the threading.
  test('reconnect receives the triggering renewLock error', async () => {
    const audit = freshAudit();
    const renewErr = new Error('write CONNECTION_ENDED');
    let received: unknown = 'NOT_CALLED';
    const deps: LockRenewalDeps = {
      renewLock: async () => { throw renewErr; },
      audit: audit.sink,
      now: () => 1000,
      setTimeout: makeFakeTimer().setTimeout,
      reconnect: async (ctx?: { error?: unknown }) => { received = ctx?.error; },
    };
    const result = await runLockRenewalTick(deps, makeState({ lastSuccessfulRenewalAt: 0 }));
    expect(result).toEqual({ kind: 'ok' });
    expect(received).toBe(renewErr);
  });

  test('a reconnect throw is swallowed — tick still returns ok (no unhandledRejection class)', async () => {
    const audit = freshAudit();
    const deps: LockRenewalDeps = {
      renewLock: async () => { throw new Error('Connection terminated unexpectedly'); },
      audit: audit.sink,
      now: () => 1000,
      setTimeout: makeFakeTimer().setTimeout,
      reconnect: async () => { throw new Error('reconnect failed: EHOSTUNREACH'); },
    };
    const state = makeState({ lastSuccessfulRenewalAt: 0 });
    const result = await runLockRenewalTick(deps, state);
    expect(result).toEqual({ kind: 'ok' });
  });

  test('reconnect is NOT called on a successful renewal', async () => {
    let reconnectCalls = 0;
    const deps: LockRenewalDeps = {
      renewLock: async () => true,
      audit: freshAudit().sink,
      now: () => 1000,
      setTimeout: makeFakeTimer().setTimeout,
      reconnect: async () => { reconnectCalls++; },
    };
    const result = await runLockRenewalTick(deps, makeState({ lastSuccessfulRenewalAt: 500 }));
    expect(result).toEqual({ kind: 'ok' });
    expect(reconnectCalls).toBe(0);
  });

  test('reconnect IS called on a deadline DEFERRAL (CEO-F3: next tick needs a live pool)', async () => {
    let reconnectCalls = 0;
    const deps: LockRenewalDeps = {
      renewLock: async () => { throw new Error('write CONNECTION_ENDED'); },
      audit: freshAudit().sink,
      // Past the soft deadline (25000) but inside hardEvict (60000):
      // primary throw → verify throw → DEFER, reconnect-once for next tick.
      now: () => 30_000,
      setTimeout: makeFakeTimer().setTimeout,
      reconnect: async () => { reconnectCalls++; },
    };
    const state = makeState({ lastSuccessfulRenewalAt: 0 });
    const result = await runLockRenewalTick(deps, state);
    expect(result).toEqual({ kind: 'ok' }); // deferred, not aborted
    expect(reconnectCalls).toBe(1);
  });

  test('reconnect is NOT called when the tick gives up past hardEvictMs', async () => {
    let reconnectCalls = 0;
    const deps: LockRenewalDeps = {
      renewLock: async () => { throw new Error('write CONNECTION_ENDED'); },
      audit: freshAudit().sink,
      now: () => 61_000, // past hardEvict (60000) → gave_up
      setTimeout: makeFakeTimer().setTimeout,
      reconnect: async () => { reconnectCalls++; },
    };
    const state = makeState({ lastSuccessfulRenewalAt: 0 });
    const result = await runLockRenewalTick(deps, state);
    expect(result).toMatchObject({ kind: 'should_abort', reason: 'lock-renewal-failed' });
    expect(reconnectCalls).toBe(0); // pointless to reconnect when we're giving up the lock
  });
});

// =============================================================================
// v0.46 (#4145) — failure-cause classification + starvation telemetry
// =============================================================================

describe('runLockRenewalTick: telemetry (issue #4145)', () => {
  test('case 13a — race timeout classifies as call-timeout (named error, not message-sniff)', async () => {
    const audit = freshAudit();
    const timer = makeFakeTimer();
    const deps: LockRenewalDeps = {
      renewLock: () => new Promise<boolean>(() => { /* hangs forever */ }),
      audit: audit.sink,
      now: () => 61_000, // past hardEvict so the verify-throws path aborts
      setTimeout: timer.setTimeout,
    };
    const state = makeState({ lastSuccessfulRenewalAt: 0 });
    const pending = runLockRenewalTick(deps, state);
    timer.runAll(); // fire the PRIMARY race timeout
    await new Promise(r => globalThis.setTimeout(r, 0)); // let the catch schedule the verify
    timer.runAll(); // fire the VERIFY race timeout
    const result = await pending;
    expect(result).toMatchObject({ kind: 'should_abort', cause: 'call-timeout' });
  });

  test('case 13b — audit failure events carry cause + lateness + overlap_skips ctx (primary + deferred verify)', async () => {
    const ctxLog: unknown[] = [];
    const audit: LockRenewalAuditSinkLike = {
      logFailure: (_id, _name, _attempt, _err, ctx) => { ctxLog.push(ctx); },
      logSuccessAfterFailure: () => { /* noop */ },
      logGaveUp: () => { /* noop */ },
    };
    const deps: LockRenewalDeps = {
      renewLock: async () => { throw new Error('outage'); },
      audit,
      // lateness = max(0, 20000 - 0 - 15000) = 5000. 20000 + 15000 >= 25000
      // → the verify fires (and also throws) → a second, deferred failure.
      now: () => 20_000,
      setTimeout: makeFakeTimer().setTimeout,
      loadSnapshot: () => ({ load1: 28.5, cores: 32 }),
    };
    const state = makeState({ lastSuccessfulRenewalAt: 0, overlapSkips: 3 });
    const result = await runLockRenewalTick(deps, state);
    expect(result).toEqual({ kind: 'ok' });
    expect(ctxLog).toHaveLength(2);
    expect(ctxLog[0]).toMatchObject({
      cause: 'refused',
      lateness_ms: 5_000,
      overlap_skips: 3,
      load1: 28.5,
      cores: 32,
    });
    expect(ctxLog[1]).toMatchObject({ cause: 'refused', deadline_deferred: true });
    expect(state.consecutiveFailures).toBe(2); // primary + verify each count
  });

  test('case 13c — a THROWING loadSnapshot never breaks the tick (CEO-F2)', async () => {
    const audit = freshAudit();
    const deps: LockRenewalDeps = {
      renewLock: async () => { throw new Error('outage'); },
      audit: audit.sink,
      now: () => 61_000, // past hardEvict → the verify-throws path aborts
      setTimeout: makeFakeTimer().setTimeout,
      loadSnapshot: () => { throw new Error('os.loadavg exploded'); },
    };
    const state = makeState({ lastSuccessfulRenewalAt: 0 });
    const result = await runLockRenewalTick(deps, state);
    // Still aborts cleanly; load fields simply absent.
    expect(result).toMatchObject({ kind: 'should_abort', cause: 'refused' });
    expect('load1' in result).toBe(false);
  });

  test('case 13d — onRenewalSuccess fires on success (R2-9 histogram reset hook) and its throw is swallowed', async () => {
    let resets = 0;
    const okDeps: LockRenewalDeps = {
      renewLock: async () => true,
      audit: freshAudit().sink,
      now: () => 1000,
      setTimeout: makeFakeTimer().setTimeout,
      onRenewalSuccess: () => { resets++; },
    };
    expect(await runLockRenewalTick(okDeps, makeState())).toEqual({ kind: 'ok' });
    expect(resets).toBe(1);

    const throwingDeps: LockRenewalDeps = {
      ...okDeps,
      onRenewalSuccess: () => { throw new Error('histogram on fire'); },
    };
    expect(await runLockRenewalTick(throwingDeps, makeState())).toEqual({ kind: 'ok' });
  });

  test('case 13e — lateness baseline advances every tick (coalesced-timer semantics)', async () => {
    let nowMs = 15_000; // exactly one interval after the seeded baseline (0)
    const deps: LockRenewalDeps = {
      renewLock: async () => true,
      audit: freshAudit().sink,
      now: () => nowMs,
      setTimeout: makeFakeTimer().setTimeout,
    };
    const state = makeState();
    await runLockRenewalTick(deps, state);
    expect(state.lastTickFiredAt).toBe(15_000); // baseline advanced

    // Next tick fires 61s later (starved past hardEvict — verify throws too):
    // lateness = 76000 - 15000 - 15000 = 46000 attributes the miss to LOCAL
    // starvation; sinceLastSuccess = 76000 - 15000 = 61000 >= 60000 → abort.
    nowMs = 76_000;
    const failDeps: LockRenewalDeps = { ...deps, renewLock: async () => { throw new Error('x'); } };
    const result = await runLockRenewalTick(failDeps, state);
    expect(result).toMatchObject({ kind: 'should_abort', latenessMs: 46_000, sinceLastSuccessMs: 61_000 });
  });
});

describe('runLockRenewalTick: per-call cancellation signal (issue #6)', () => {
  test('renewLock receives a live AbortSignal; not aborted on the happy path', async () => {
    const audit = freshAudit();
    const timer = makeFakeTimer();
    let seenSignal: AbortSignal | undefined;
    const deps: LockRenewalDeps = {
      renewLock: async (_id, _tok, _dur, opts) => {
        seenSignal = opts?.signal;
        return true;
      },
      audit: audit.sink,
      now: () => 1000,
      setTimeout: timer.setTimeout,
    };
    const result = await runLockRenewalTick(deps, makeState());
    expect(result.kind).toBe('ok');
    expect(seenSignal).toBeInstanceOf(AbortSignal);
    expect(seenSignal!.aborted).toBe(false);
  });

  test('hung renewLock: the timeout aborts the per-call signal so the query is cancelled, not orphaned', async () => {
    const audit = freshAudit();
    const timer = makeFakeTimer();
    let seenSignal: AbortSignal | undefined;
    const deps: LockRenewalDeps = {
      renewLock: (_id, _tok, _dur, opts) => {
        seenSignal = opts?.signal;
        return new Promise<boolean>(() => { /* hangs forever — abandoned racer */ });
      },
      audit: audit.sink,
      now: () => 1000,
      setTimeout: timer.setTimeout,
    };
    // Renewed recently: the timeout counts as failure #1, not a give-up.
    const state = makeState({ lastSuccessfulRenewalAt: 1000 });
    const tick = runLockRenewalTick(deps, state);
    // The race installs the timeout synchronously; firing it must abort the
    // per-call signal (this is what releases the checked-out pool slot).
    timer.runAll();
    const result = await tick;
    expect(result.kind).toBe('ok'); // counter bumped, deadline not yet crossed
    expect(state.consecutiveFailures).toBe(1);
    expect(seenSignal).toBeInstanceOf(AbortSignal);
    expect(seenSignal!.aborted).toBe(true);
  });
});