Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
5.3 KB
Raw Blame
/**
 * #7263: the background doctor-refresh job must not hold a one-slot worker. It queues behind sync, the autopilot
 * cycles, global maintenance and loops extract (a numerically higher priority), stops starting checks at an overall
 * budget well under its hourly slot and still completes, recording what it measured, and an operator can turn it off.
 */
import { afterAll, beforeAll, beforeEach, describe, expect, test } from 'bun:test';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { MinionQueue } from '../src/core/minions/queue.ts';
import { MinionWorker } from '../src/core/minions/worker.ts';
import { DEFAULT_LONG_JOB_NAMES } from '../src/core/minions/lanes.ts';
import { LAST_KNOWN_OP, REFRESH_BUDGET_MS, REFRESH_JOB, queueRefresh, runDoctorRefresh } from '../src/commands/doctor/last-known.ts';
import { waitFor } from './helpers/wait-for.ts';

let engine: PGLiteEngine;
let queue: MinionQueue;

beforeAll(async () => {
  engine = new PGLiteEngine();
  await engine.connect({});
  await engine.initSchema();
  queue = new MinionQueue(engine);
}, 60_000);

afterAll(async () => {
  await engine.disconnect();
});

beforeEach(async () => {
  await engine.executeRaw('DELETE FROM minion_jobs');
  await engine.executeRaw('DELETE FROM op_checkpoints WHERE op = $1', [LAST_KNOWN_OP]);
  await engine.executeRaw("DELETE FROM config WHERE key = 'doctor.background_refresh'");
});

async function claimOrder(): Promise<number[]> {
  const rows = await engine.executeRaw<{ id: number }>('SELECT id FROM minion_jobs WHERE started_at IS NOT NULL ORDER BY started_at, id');
  return rows.map(r => Number(r.id));
}

/** An engine whose every SQL statement takes 100 ms, so each doctor entry is slow and the full refresh takes far longer than its budget. */
function slowEngine(inner: PGLiteEngine): PGLiteEngine {
  return new Proxy(inner, {
    get(target, prop, receiver) {
      const value = Reflect.get(target, prop, receiver);
      if (typeof value !== 'function') return value;
      if (prop === 'executeRaw' || prop === 'executeRawDirect') {
        return async (...args: unknown[]) => { await new Promise(r => setTimeout(r, 100)); return value.apply(target, args); };
      }
      return value.bind(target);
    },
  });
}

describe('doctor-refresh on a one-slot worker (#7263)', () => {
  test('a sync queued after a refresh is claimed first, and the refresh runs once the slot is free', async () => {
    const refresh = await queueRefresh(engine);
    expect(refresh).not.toBeNull();
    const sync = await queue.add('sync', { sourceId: 'source-a' });
    const worker = new MinionWorker(engine, { concurrency: 1, pollInterval: 20, healthCheckInterval: 0, stalledInterval: 60_000 });
    worker.register(REFRESH_JOB, async () => ({ ok: true }));
    worker.register('sync', async () => ({ ok: true }));
    const running = worker.start();
    try {
      await waitFor(async () => (await queue.getJob(refresh!.id))?.status === 'completed', { timeoutMs: 10_000, label: 'refresh completed' });
    } finally {
      worker.stop();
      await running;
    }
    expect(await claimOrder()).toEqual([sync.id, refresh!.id]);
  }, 30_000);

  test('the refresh queues behind sync, the autopilot cycles, global maintenance and loops extract, with a timeout over its budget', async () => {
    const job = (await queue.getJob((await queueRefresh(engine))!.id))!;
    expect(job.priority).toBeGreaterThan(5);
    expect(job.timeout_ms).toBeGreaterThan(REFRESH_BUDGET_MS);
    expect(job.timeout_ms).toBeLessThanOrEqual(30 * 60_000);
    expect(REFRESH_BUDGET_MS).toBeLessThanOrEqual(10 * 60_000);
  });

  test('a two-slot worker treats the refresh as a long job', () => {
    expect(DEFAULT_LONG_JOB_NAMES).toContain(REFRESH_JOB);
  });

  test('doctor.background_refresh false queues nothing', async () => {
    await engine.setConfig('doctor.background_refresh', 'false');
    expect(await queueRefresh(engine)).toBeNull();
    expect(await engine.executeRaw('SELECT id FROM minion_jobs WHERE name = $1', [REFRESH_JOB])).toHaveLength(0);
  });
});

describe('doctor-refresh overall budget (#7263)', () => {
  test('a refresh whose entries are slow stops at its budget, records what it measured and completes', async () => {
    const budgetMs = 3_000;
    const job = await queue.add(REFRESH_JOB, {}, { queue: 'default', max_attempts: 1 });
    const worker = new MinionWorker(engine, { concurrency: 1, pollInterval: 20, healthCheckInterval: 0, stalledInterval: 60_000 });
    worker.register(REFRESH_JOB, async () => runDoctorRefresh(slowEngine(engine), budgetMs));
    const running = worker.start();
    try {
      await waitFor(async () => ['completed', 'failed', 'dead'].includes((await queue.getJob(job.id))?.status ?? ''), { timeoutMs: 30_000, label: 'refresh settled' });
    } finally {
      worker.stop();
      await running;
    }
    const done = (await queue.getJob(job.id))!;
    expect(done.status).toBe('completed');
    const result = done.result as { recorded: number; unfinished: string[]; elapsed_ms: number };
    expect(result.unfinished.length).toBeGreaterThan(0);
    expect(result.recorded).toBeGreaterThan(0);
    expect(result.elapsed_ms).toBeLessThan(budgetMs + 5_000);
    const [{ n }] = await engine.executeRaw<{ n: number }>('SELECT count(*)::int AS n FROM op_checkpoints WHERE op = $1', [LAST_KNOWN_OP]);
    expect(n).toBeGreaterThan(0);
  }, 60_000);
});