Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
12.5 KB
Raw Blame
/**
 * Jobs fix wave (upstream issues #2/#3) — migration v128
 * (minion_jobs_timeout_backfill_and_duplicate_cycle_cleanup).
 *
 * Pinned contracts:
 * 1. v128 exists in MIGRATIONS with the canonical name, idempotent flag, and
 *    one engine-agnostic sql block (no sqlFor split).
 * 2. Statement 1 backfills NULL timeout_ms for the 8 long-lane handler names
 *    across ALL five non-terminal statuses — and ONLY those rows: terminal
 *    rows, unmapped names, and explicit budgets are untouched. Active rows
 *    get timeout_ms only (never a timeout_at stamp — that would arm the
 *    tighter 1x handleTimeouts kill mid-flight; the 2x wall-clock bound is
 *    the intended repair).
 * 3. Statement 2 cancels all-but-newest ticker-keyed waiting autopilot
 *    cycles per (name, queue, source scope). Manual submissions (no ticker
 *    idempotency-key prefix) and parented rows are never touched; distinct
 *    sources each keep their newest row; cancelled rows carry error_text.
 * 4. Ledger round-trip on a waiting fixture: rewind to 127 → runMigrations
 *    applies v128; re-run is 0 applied. The active-row matrix runs v128 SQL
 *    directly because later authority cutovers require active jobs drained.
 *    AND (Codex C9) SQL-level idempotency: re-executing the
 *    v128 SQL directly changes zero rows — the ledger check alone only
 *    proves version bookkeeping.
 * 5. Empty table → 0-row no-op.
 */

import { describe, test, expect, beforeAll, afterAll, beforeEach } from 'bun:test';
import type { BrainEngine } from '../src/core/engine.ts';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { MinionQueue } from '../src/core/minions/queue.ts';
import { MIGRATIONS, LATEST_VERSION, runMigrations } from '../src/core/migrate.ts';

let engine: PGLiteEngine;
let queue: MinionQueue;
let workerBackedQueue: MinionQueue;

const V128_SQL = MIGRATIONS.find(m => m.version === 128)?.sql ?? '';

/** PGLite's prepared-statement path rejects multi-command SQL; the migration
 *  runner uses a different execution path. For the direct SQL-rerun check,
 *  split on statement terminators (the blob's only two `;` are terminators). */
const V128_STATEMENTS = V128_SQL.split(';').map(s => s.trim()).filter(Boolean);

async function execV128Directly(): Promise<void> {
  for (const stmt of V128_STATEMENTS) {
    await engine.executeRaw(stmt);
  }
}

beforeAll(async () => {
  engine = new PGLiteEngine();
  await engine.connect({ database_url: '' }); // in-memory
  await engine.initSchema();
  queue = new MinionQueue(engine);
  const workerBackedEngine = new Proxy(engine, {
    get(target, prop) {
      if (prop === 'kind') return 'postgres';
      const value = Reflect.get(target, prop, target);
      return typeof value === 'function' ? value.bind(target) : value;
    },
  }) as unknown as BrainEngine;
  workerBackedQueue = new MinionQueue(workerBackedEngine);
});

afterAll(async () => {
  await engine.disconnect();
});

beforeEach(async () => {
  await engine.executeRaw('DELETE FROM minion_jobs');
});

/** Force a row into a specific historical shape (add() stamps budgets and
 *  every row starts 'waiting', so legacy states are seeded by direct UPDATE —
 *  the same trick the wall-clock tests use). */
async function forceRow(id: number, sets: string, params: unknown[] = []): Promise<void> {
  await engine.executeRaw(`UPDATE minion_jobs SET ${sets} WHERE id = $${params.length + 1}`, [...params, id]);
}

async function snapshot(): Promise<Array<Record<string, unknown>>> {
  return engine.executeRaw<Record<string, unknown>>(
    `SELECT id, name, status, timeout_ms::text AS timeout_ms,
            timeout_at IS NULL AS timeout_at_null, error_text
       FROM minion_jobs ORDER BY id`,
  );
}

describe('migration v128 — structure', () => {
  test('exists with canonical name, idempotent flag, engine-agnostic sql', () => {
    const v128 = MIGRATIONS.find(m => m.version === 128);
    expect(v128).toBeDefined();
    expect(v128?.name).toBe('minion_jobs_timeout_backfill_and_duplicate_cycle_cleanup');
    expect(v128?.idempotent).toBe(true);
    expect(v128?.sqlFor).toBeUndefined();
    expect(LATEST_VERSION).toBeGreaterThanOrEqual(128);
  });

  test('statement 1 covers all five non-terminal statuses and never stamps timeout_at', () => {
    expect(V128_SQL).toContain(`status IN ('waiting','active','delayed','waiting-children','paused')`);
    expect(V128_SQL).not.toContain('SET timeout_at');
  });

  test('statement 2 is restricted to ticker idempotency-key prefixes and top-level rows', () => {
    expect(V128_SQL).toContain(`idempotency_key LIKE 'autopilot-cycle:%'`);
    expect(V128_SQL).toContain(`idempotency_key LIKE 'autopilot-global:%'`);
    expect(V128_SQL).toContain('parent_job_id IS NULL');
  });
});

describe('migration v128 — backfill + cleanup semantics (PGLite)', () => {
  test('full matrix: backfill hits legacy non-terminal mapped rows only; cleanup keeps newest ticker row per scope', async () => {
    // --- Backfill fixtures (statement 1) ---
    // Legacy waiting subagent: NULL budget → 30min.
    const legacyWaiting = await queue.add('subagent', {}, undefined, { allowProtectedSubmit: true });
    await forceRow(legacyWaiting.id, `timeout_ms = NULL, timeout_at = NULL`);
    // Legacy ACTIVE autopilot-cycle mid-flight: NULL budget → 30min, but
    // timeout_at must STAY NULL (Codex C7 — no 1x kill armed mid-flight).
    const legacyActive = await queue.add('autopilot-cycle', { source_id: 'src-active' }, {
      idempotency_key: 'autopilot-cycle:src-active:slot-0',
    });
    await forceRow(legacyActive.id,
      `timeout_ms = NULL, timeout_at = NULL, status = 'active', claim_generation = claim_generation + 1,
       lock_token = 'legacy-worker', lock_until = now() + interval '30 seconds',
       started_at = now() - interval '10 minutes'`);
    // Terminal row: NULL budget stays NULL (completed is not rescued).
    const completed = await queue.add('subagent', { done: true }, undefined, { allowProtectedSubmit: true });
    await forceRow(completed.id, `timeout_ms = NULL, status = 'completed', finished_at = now()`);
    // Unmapped name: stays NULL.
    const unmapped = await queue.add('sync', {});
    expect(unmapped.timeout_ms).toBeNull();
    // Explicit budget: untouched.
    const explicit = await workerBackedQueue.add('embed-backfill', { sourceId: 'x' }, { timeout_ms: 5000 });

    // --- Cleanup fixtures (statement 2) ---
    // Three ticker-keyed waiting cycles for ONE source, staggered ages —
    // newest must survive, older two cancelled.
    const dupOld = await queue.add('autopilot-cycle', { source_id: 'src-dup' }, {
      idempotency_key: 'autopilot-cycle:src-dup:slot-1',
    });
    await forceRow(dupOld.id, `timeout_ms = NULL, created_at = now() - interval '3 hours'`);
    const dupMid = await queue.add('autopilot-cycle', { source_id: 'src-dup' }, {
      idempotency_key: 'autopilot-cycle:src-dup:slot-2',
    });
    await forceRow(dupMid.id, `timeout_ms = NULL, created_at = now() - interval '2 hours'`);
    const dupNew = await queue.add('autopilot-cycle', { source_id: 'src-dup' }, {
      idempotency_key: 'autopilot-cycle:src-dup:slot-3',
    });
    await forceRow(dupNew.id, `timeout_ms = NULL, created_at = now() - interval '1 hour'`);
    // A DIFFERENT source keeps its own newest row (scope isolation).
    const otherSrc = await queue.add('autopilot-cycle', { source_id: 'src-other' }, {
      idempotency_key: 'autopilot-cycle:src-other:slot-1',
    });
    await forceRow(otherSrc.id, `created_at = now() - interval '4 hours'`);
    // Global maintenance duplicates (NULL source scope) dedupe independently.
    const globalOld = await queue.add('autopilot-global-maintenance', {}, {
      idempotency_key: 'autopilot-global:slot-1',
    });
    await forceRow(globalOld.id, `created_at = now() - interval '2 hours'`);
    const globalNew = await queue.add('autopilot-global-maintenance', {}, {
      idempotency_key: 'autopilot-global:slot-2',
    });
    // Manual submission: same name + source as the dup scope, custom phases,
    // NO ticker key — must never be touched (Codex C5).
    const manual = await queue.add('autopilot-cycle', { source_id: 'src-dup', phases: ['sync'] });
    await forceRow(manual.id, `created_at = now() - interval '5 hours'`);
    // Ticker-mimicking key but camelCase sourceId payload: the ticker never
    // writes sourceId, so this row is not ticker-provenance — the cleanup's
    // sourceId-IS-NULL guard must leave it alone even though its key matches
    // the prefix (adversarial-review tightening: without the guard, distinct
    // sourceId scopes would collapse into the empty source_id group).
    const camelMimic = await queue.add('autopilot-cycle', { sourceId: 'camel-src' }, {
      idempotency_key: 'autopilot-cycle:camel-mimic:slot-1',
    });
    await forceRow(camelMimic.id, `created_at = now() - interval '7 hours'`);
    // Parented row with a ticker-looking key: guarded by parent_job_id IS NULL.
    // Parent is seeded by direct UPDATE — add()'s parent_job_id opt would flip
    // the parent row to 'waiting-children' and pollute the dup-scope fixtures.
    const parented = await queue.add('autopilot-cycle', { source_id: 'src-dup' }, {
      idempotency_key: 'autopilot-cycle:src-dup:slot-child',
    });
    await forceRow(parented.id,
      `created_at = now() - interval '6 hours', parent_job_id = $1`, [dupNew.id]);

    // Keep the active-row contract local to v128; the v149 authority
    // cutover independently requires active jobs to have drained.
    await execV128Directly();

    const rows = await engine.executeRaw<{
      id: number; status: string; timeout_ms: string | null;
      timeout_at: string | null; error_text: string | null;
    }>(`SELECT id, status, timeout_ms::text AS timeout_ms, timeout_at::text AS timeout_at, error_text
          FROM minion_jobs`);
    const byId = new Map(rows.map(r => [Number(r.id), r]));

    // Statement 1 assertions.
    expect(Number(byId.get(legacyWaiting.id)!.timeout_ms)).toBe(1_800_000);
    expect(Number(byId.get(legacyActive.id)!.timeout_ms)).toBe(1_800_000);
    expect(byId.get(legacyActive.id)!.timeout_at).toBeNull();       // C7: no 1x kill armed
    expect(byId.get(legacyActive.id)!.status).toBe('active');
    expect(byId.get(completed.id)!.timeout_ms).toBeNull();          // terminal untouched
    expect(byId.get(unmapped.id)!.timeout_ms).toBeNull();           // not in map
    expect(Number(byId.get(explicit.id)!.timeout_ms)).toBe(5000);   // explicit wins

    // Statement 2 assertions.
    expect(byId.get(dupNew.id)!.status).toBe('waiting');            // newest survives
    expect(byId.get(dupOld.id)!.status).toBe('cancelled');
    expect(byId.get(dupMid.id)!.status).toBe('cancelled');
    expect(byId.get(dupOld.id)!.error_text).toContain('superseded duplicate autopilot cycle');
    expect(byId.get(otherSrc.id)!.status).toBe('waiting');          // other source keeps its one row
    expect(byId.get(globalNew.id)!.status).toBe('waiting');
    expect(byId.get(globalOld.id)!.status).toBe('cancelled');
    expect(byId.get(manual.id)!.status).toBe('waiting');            // manual never touched (C5)
    expect(byId.get(manual.id)!.error_text).toBeNull();
    expect(byId.get(parented.id)!.status).toBe('waiting');          // parented never touched
    expect(byId.get(camelMimic.id)!.status).toBe('waiting');        // camelCase sourceId never swept

    // SQL-level idempotency (Codex C9): re-execute the v128 SQL directly and
    // prove zero rows change — version bookkeeping alone can't show this.
    const before = await snapshot();
    await execV128Directly();
    const after = await snapshot();
    expect(after).toEqual(before);

    // Sanity on the split used above: exactly the two statements shipped.
    expect(V128_STATEMENTS.length).toBe(2);
  }, 30_000);

  test('migration runner backfills a waiting fixture and records an idempotent ledger', async () => {
    const waiting = await queue.add('subagent', {}, undefined, { allowProtectedSubmit: true });
    await forceRow(waiting.id, `timeout_ms = NULL, timeout_at = NULL`);
    await engine.setConfig('version', '127');

    const res = await runMigrations(engine);
    expect(res.applied).toBeGreaterThanOrEqual(1);
    expect(await engine.getConfig('version')).toBe(String(LATEST_VERSION));
    expect((await queue.getJob(waiting.id))?.timeout_ms).toBe(1_800_000);
    expect((await runMigrations(engine)).applied).toBe(0);
  });

  test('empty table: v128 SQL is a 0-row no-op', async () => {
    const before = await snapshot();
    expect(before).toEqual([]);
    await execV128Directly();
    expect(await snapshot()).toEqual([]);
  });
});