Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
6.8 KB
Raw Blame
/**
 * W0 fix-wave (Tier-1 #7) — started_at is reset on every AUTOMATIC re-run
 * path, so handleWallClockTimeouts (anchored on now() - started_at) measures
 * per-attempt runtime, never time a job spent parked in backoff.
 *
 * Pre-fix, only the manual `jobs retry` path cleared started_at (its
 * docstring documented the bug); the four automatic paths — failJob's
 * delayed branch, handleStalled's requeue, promoteDelayed, and
 * releaseLeaseFullJob (the 4th site, found in adversarial verification) —
 * all preserved the FIRST claim's timestamp, so an exponential-backoff job
 * could be dead-lettered by the wall-clock sweep before executing a single
 * line of its retry attempt.
 */

import { test, expect, beforeAll, afterAll, beforeEach } from 'bun:test';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { MinionQueue } from '../src/core/minions/queue.ts';

let engine: PGLiteEngine;
let queue: MinionQueue;

beforeAll(async () => {
  engine = new PGLiteEngine();
  await engine.connect({ database_url: '' });
  await engine.initSchema();
  queue = new MinionQueue(engine);
});

afterAll(async () => {
  await engine.disconnect();
});

beforeEach(async () => {
  await engine.executeRaw('DELETE FROM minion_jobs');
});

async function jobStatus(id: number): Promise<string> {
  const rows = await engine.executeRaw<{ status: string }>(
    `SELECT status FROM minion_jobs WHERE id = $1`, [id],
  );
  return rows[0]!.status;
}

async function startedAtOf(id: number): Promise<string | null> {
  const rows = await engine.executeRaw<{ started_at: string | null }>(
    `SELECT started_at::text AS started_at FROM minion_jobs WHERE id = $1`, [id],
  );
  return rows[0]?.started_at ?? null;
}

async function addAndClaim(name = 'noop'): Promise<{ id: number; token: string }> {
  const job = await queue.add(name, {});
  const token = `tok-${Math.random().toString(36).slice(2)}`;
  const claimed = await queue.claim(token, 30_000, 'default', [name]);
  expect(claimed?.id).toBe(job.id);
  expect(await startedAtOf(job.id)).not.toBeNull();
  return { id: job.id, token };
}

test("failJob's delayed branch clears started_at (terminal branches keep it)", async () => {
  const { id, token } = await addAndClaim();
  const delayed = await queue.failJob(id, token, 'boom', 'delayed', 50);
  expect(delayed?.status).toBe('delayed');
  expect(await startedAtOf(id)).toBeNull();

  // Terminal failure keeps started_at (it feeds duration accounting).
  const { id: id2, token: token2 } = await addAndClaim();
  const dead = await queue.failJob(id2, token2, 'boom', 'dead');
  expect(dead?.status).toBe('dead');
  expect(await startedAtOf(id2)).not.toBeNull();
});

test("handleStalled's requeue branch clears started_at", async () => {
  const { id } = await addAndClaim();
  await engine.executeRaw(
    `UPDATE minion_jobs SET lock_until = now() - interval '30 seconds' WHERE id = $1`, [id],
  );
  const { requeued } = await queue.handleStalled();
  expect(requeued.map(j => j.id)).toContain(id);
  expect(await startedAtOf(id)).toBeNull();
});

test('releaseLeaseFullJob clears started_at (lease bounce burns no wall-clock)', async () => {
  const { id, token } = await addAndClaim();
  const released = await queue.releaseLeaseFullJob(id, token, 'rate lease full (8/8)', 50);
  expect(released?.status).toBe('delayed');
  expect(await startedAtOf(id)).toBeNull();
});

test('promoteDelayed defensively clears any stale started_at', async () => {
  const job = await queue.add('noop', {});
  await engine.executeRaw(
    `UPDATE minion_jobs
        SET status = 'delayed', delay_until = now() - interval '1 second',
            started_at = now() - interval '1 hour'
      WHERE id = $1`, [job.id],
  );
  const promoted = await queue.promoteDelayed();
  expect(promoted.map(j => j.id)).toContain(job.id);
  expect(await startedAtOf(job.id)).toBeNull();
});

test('end-to-end: a retried job survives the wall-clock sweep on its fresh attempt', async () => {
  const { id, token } = await addAndClaim();
  // Simulate a long first attempt: backdate started_at far past the sweep
  // threshold (no timeout_ms → threshold = lockDurationMs * 2 * max_stalled).
  await engine.executeRaw(
    `UPDATE minion_jobs SET started_at = now() - interval '1 hour' WHERE id = $1`, [id],
  );
  await queue.failJob(id, token, 'first attempt failed', 'delayed', 0);
  const promoted = await queue.promoteDelayed();
  expect(promoted.map(j => j.id)).toContain(id);

  const token2 = 'tok-second-attempt';
  const reclaimed = await queue.claim(token2, 30_000, 'default', ['noop']);
  expect(reclaimed?.id).toBe(id);

  // Pre-fix: started_at still said now()-1h → 3.6M ms > 30_000*2*max_stalled
  // and the very next sweep dead-lettered the fresh attempt. Post-fix the
  // re-claim stamped a fresh started_at, so the sweep spares it.
  const killed = await queue.handleWallClockTimeouts(30_000);
  expect(killed.map(j => j.id)).not.toContain(id);

  // Negative control — the sweep itself still works: backdate the ACTIVE
  // attempt past the threshold and it dies.
  await engine.executeRaw(
    `UPDATE minion_jobs SET started_at = now() - interval '1 hour' WHERE id = $1`, [id],
  );
  const killed2 = await queue.handleWallClockTimeouts(30_000);
  expect(killed2.map(j => j.id)).toContain(id);
});

test('red-team 5th path: a re-claimed aggregator parent survives the wall-clock sweep', async () => {
  // Parent parked in waiting-children for LONGER than the wall-clock
  // threshold while its child runs; every unblock path (killJobs,
  // completeJob resolve, failJob branches, cancelJob, retroactive sweep,
  // resolveParent) must clear started_at, or claim()'s COALESCE keeps the
  // attempt-1 anchor and the sweep dead-letters the aggregation attempt
  // before it executes a line.
  const parent = await queue.add('agg-wallclock', {});
  const child = await queue.add('child-wc', {}, { parent_job_id: parent.id, max_stalled: 1 });
  // Parent claimed long ago (attempt-1 anchor), then parked waiting-children.
  await engine.executeRaw(
    `UPDATE minion_jobs SET status = 'waiting-children', started_at = now() - interval '1 hour' WHERE id = $1`, [parent.id],
  );
  // Child stalls to death → killJobs unblocks the parent.
  await queue.claim('tok-wc', 30_000, 'default', ['child-wc', 'agg-wallclock']);
  await engine.executeRaw(
    `UPDATE minion_jobs SET lock_until = now() - interval '30 seconds' WHERE id = $1`, [child.id],
  );
  await queue.handleStalled();
  expect(await jobStatus(parent.id)).toBe('waiting');
  // The fix: unblock cleared the stale anchor.
  expect(await startedAtOf(parent.id)).toBeNull();

  // Parent re-claims for its aggregation attempt → fresh anchor → survives.
  const reclaimed = await queue.claim('tok-wc2', 30_000, 'default', ['agg-wallclock', 'child-wc']);
  expect(reclaimed?.id).toBe(parent.id);
  const killed = await queue.handleWallClockTimeouts(30_000);
  expect(killed.map(j => j.id)).not.toContain(parent.id);
});