Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
10.7 KB
Raw Blame
/**
 * F1 — SyncLockBusyError classifies as skipped-not-failed in BOTH consumers.
 *
 * The real seams (no stubs — a genuinely HELD per-source sync lock in PGLite
 * makes the real performSync throw the real SyncLockBusyError):
 *
 *   - src/commands/sync.ts:performSync wraps LockUnavailableError from
 *     withRefreshingLock into SyncLockBusyError(formatLockBusyMessage(...)).
 *   - Consumer 1 — the Minion 'sync' handler registered by
 *     registerBuiltinHandlers (src/commands/jobs.ts): catches
 *     SyncLockBusyError and RETURNS
 *     { skipped: true, reason: 'sync_in_progress', source_id } so the worker
 *     marks the job 'completed', never failed/dead.
 *   - Consumer 2 — runPhaseSync (src/core/cycle.ts, private; exercised
 *     through the real runCycle with phases: ['sync']): catches
 *     SyncLockBusyError and reports
 *     { phase: 'sync', status: 'skipped', details: { syncStatus: 'lock_busy' } }.
 *
 * Anti-vacuity: each consumer also has a red control — a NON-lock error
 * (vanished repo dir) propagates as a real failure ('dead' job / 'fail'
 * phase + 'failed' cycle), proving the classification CAN differ.
 *
 * Plus src/core/sync-lock.ts:formatLockBusyMessage — rich message names
 * holder pid/host and the `--break-lock --source` remediation; degrades to
 * the legacy message (no crash) when inspectLock throws.
 */
import { describe, test, expect, beforeAll, afterAll, beforeEach } from 'bun:test';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { resetPgliteState } from './helpers/reset-pglite.ts';
import { withEnv, emptyHome } from './helpers/with-env.ts';
import { MinionQueue } from '../src/core/minions/queue.ts';
import { MinionWorker } from '../src/core/minions/worker.ts';
import type { MinionJob } from '../src/core/minions/types.ts';
import { registerBuiltinHandlers } from '../src/commands/jobs.ts';
import { runCycle } from '../src/core/cycle.ts';
import { formatLockBusyMessage } from '../src/core/sync-lock.ts';
import { tryAcquireDbLock, syncLockId } from '../src/core/db-lock.ts';
import type { BrainEngine } from '../src/core/engine.ts';
import { mkdtempSync, writeFileSync, rmSync } from 'fs';
import { execSync } from 'child_process';
import { tmpdir, hostname } from 'os';
import { join } from 'path';

let engine: PGLiteEngine;
// MinionQueue.ensureSchema (called by worker.start()) reads config.version;
// resetPgliteState TRUNCATEs the config table, so we capture the migrated
// version once and re-seed it after every reset.
let schemaVersion: string | null = null;

beforeAll(async () => {
  engine = new PGLiteEngine();
  await engine.connect({});
  await engine.initSchema();
  schemaVersion = await engine.getConfig('version');
}, 120_000);

afterAll(async () => {
  await engine.disconnect();
});

beforeEach(async () => {
  await resetPgliteState(engine);
  if (schemaVersion) await engine.setConfig('version', schemaVersion);
});

function makeGitRepo(): string {
  const dir = mkdtempSync(join(tmpdir(), 'gbrain-f1-lock-busy-'));
  execSync('git init', { cwd: dir, stdio: 'pipe' });
  execSync('git config user.email [email protected]', { cwd: dir, stdio: 'pipe' });
  execSync('git config user.name t', { cwd: dir, stdio: 'pipe' });
  writeFileSync(join(dir, '.gitkeep'), '');
  execSync('git add -A && git commit -m init', { cwd: dir, stdio: 'pipe' });
  return dir;
}

const TERMINAL = new Set(['completed', 'failed', 'dead', 'cancelled']);

async function waitForTerminal(queue: MinionQueue, id: number, timeoutMs = 45_000): Promise<MinionJob> {
  const deadline = Date.now() + timeoutMs;
  while (Date.now() < deadline) {
    const job = await queue.getJob(id);
    if (job && TERMINAL.has(job.status)) return job;
    await new Promise((r) => setTimeout(r, 50));
  }
  throw new Error(`job ${id} did not reach a terminal state within ${timeoutMs}ms`);
}

/** Run the real builtin-handler worker until the job terminates, then stop it. */
async function runJobToTerminal(job: MinionJob, queue: MinionQueue): Promise<MinionJob> {
  const worker = new MinionWorker(engine, { pollInterval: 50 });
  await registerBuiltinHandlers(worker, engine, { quiet: true });
  const workerPromise = worker.start();
  try {
    return await waitForTerminal(queue, job.id);
  } finally {
    worker.stop();
    await workerPromise;
  }
}

// ─── Consumer 1: Minion 'sync' job handler (src/commands/jobs.ts) ──────────

describe('minion sync handler — SyncLockBusyError is skipped, not failed', () => {
  test('held per-source sync lock → job completes with { skipped: true, reason: "sync_in_progress" }', async () => {
    await withEnv({ GBRAIN_HOME: emptyHome() }, async () => {
      const repo = makeGitRepo();
      // Real lock helper, real PGLite row: gbrain-sync:jobsrc held by THIS
      // (alive) pid, so performSync's acquire fails and the takeover path
      // refuses to steal from a live holder.
      const held = await tryAcquireDbLock(engine, syncLockId('jobsrc'));
      expect(held).not.toBeNull();
      try {
        const queue = new MinionQueue(engine);
        const job = await queue.add(
          'sync',
          { repoPath: repo, sourceId: 'jobsrc', pull: false },
          { max_attempts: 1 },
        );
        const terminal = await runJobToTerminal(job, queue);

        // The classification claim: the job is done, NOT failed/dead.
        expect(terminal.status).toBe('completed');
        // Pin the real constant strings from src/commands/jobs.ts.
        expect(terminal.result).toEqual({
          skipped: true,
          reason: 'sync_in_progress',
          source_id: 'jobsrc',
        });
        expect(terminal.error_text ?? null).toBeNull();
      } finally {
        await held!.release();
      }
    });
  }, 90_000);

  test('red control: a NON-lock error still fails the job (dead at max_attempts=1)', async () => {
    await withEnv({ GBRAIN_HOME: emptyHome() }, async () => {
      const queue = new MinionQueue(engine);
      // No lock held; the repo path does not exist, so performSync acquires
      // the lock fine and then throws a plain Error from git-root discovery.
      const job = await queue.add(
        'sync',
        { repoPath: '/nonexistent-gbrain-f1-red-control', pull: false },
        { max_attempts: 1 },
      );
      const terminal = await runJobToTerminal(job, queue);

      // Classification CAN differ: this consumer marks non-lock errors dead.
      expect(terminal.status).toBe('dead');
      expect(terminal.result ?? null).toBeNull();
      expect(terminal.error_text ?? '').not.toBe('');
      expect(terminal.error_text ?? '').not.toContain('sync_in_progress');
    });
  }, 90_000);
});

// ─── Consumer 2: cycle sync phase (src/core/cycle.ts runPhaseSync) ─────────
// runPhaseSync is module-private; the real seam is runCycle with
// phases: ['sync'] against the real performSync (no stubs).

describe('cycle sync phase — SyncLockBusyError is a skip, not a phase failure', () => {
  test('held per-source sync lock → phase skipped with details.syncStatus lock_busy', async () => {
    await withEnv({ GBRAIN_HOME: emptyHome() }, async () => {
      const brainDir = makeGitRepo();
      // Register the dir as a source so resolveSourceForDir routes the sync
      // phase to this source's lock key (per-source lock routing pinned too).
      await engine.executeRaw(
        `INSERT INTO sources (id, name, local_path, config, archived, created_at)
         VALUES ($1, $2, $3, '{}'::jsonb, false, NOW())
         ON CONFLICT (id) DO UPDATE SET local_path = EXCLUDED.local_path`,
        ['cyclesrc', 'cyclesrc', brainDir],
      );
      const held = await tryAcquireDbLock(engine, syncLockId('cyclesrc'));
      expect(held).not.toBeNull();
      try {
        const report = await runCycle(engine, { brainDir, phases: ['sync'] });
        const sync = report.phases.find((p) => p.phase === 'sync');
        expect(sync).toBeDefined();
        // Pin the real constant strings from src/core/cycle.ts runPhaseSync.
        expect(sync!.status).toBe('skipped');
        expect(sync!.details.syncStatus).toBe('lock_busy');
        expect(sync!.summary).toBe('sync already in progress elsewhere — skipped');
        expect(sync!.error).toBeUndefined();
        // The skip never paints the whole cycle red.
        expect(report.status).not.toBe('failed');
      } finally {
        await held!.release();
      }
    });
  }, 90_000);

  test('red control: a NON-lock error still fails the phase (and the cycle)', async () => {
    await withEnv({ GBRAIN_HOME: emptyHome() }, async () => {
      const brainDir = makeGitRepo();
      rmSync(brainDir, { recursive: true, force: true });
      const report = await runCycle(engine, { brainDir, phases: ['sync'] });
      const sync = report.phases.find((p) => p.phase === 'sync');
      expect(sync).toBeDefined();
      // Classification CAN differ: same consumer, non-lock error → 'fail',
      // no lock_busy marker, structured error attached.
      expect(sync!.status).toBe('fail');
      expect(sync!.details.syncStatus).toBeUndefined();
      expect(sync!.error).toBeDefined();
      expect(report.status).toBe('failed');
    });
  }, 90_000);
});

// ─── formatLockBusyMessage (src/core/sync-lock.ts) ─────────────────────────

describe('formatLockBusyMessage', () => {
  test('names holder pid/host and includes --break-lock + --source remediation', async () => {
    const lockKey = syncLockId('msgsrc');
    const held = await tryAcquireDbLock(engine, lockKey);
    expect(held).not.toBeNull();
    try {
      const msg = await formatLockBusyMessage(engine, lockKey);
      expect(msg).toContain(`lock ${lockKey} held by pid ${process.pid} on ${hostname()}`);
      expect(msg).toContain('--break-lock');
      expect(msg).toContain(`gbrain sync --break-lock --source msgsrc`);
    } finally {
      await held!.release();
    }
  });

  test('degrades to the legacy message when inspectLock throws (no crash)', async () => {
    // A poisoned engine whose query path throws makes the REAL inspectLock
    // fail; formatLockBusyMessage must catch and fall back, never throw.
    const poisoned = {
      kind: 'pglite',
      db: {
        query: async () => {
          throw new Error('inspect boom');
        },
      },
    } as unknown as BrainEngine;
    const lockKey = syncLockId('msgsrc');
    const msg = await formatLockBusyMessage(poisoned, lockKey);
    expect(msg).toBe(
      `Another sync is in progress (lock ${lockKey} held). ` +
        `Wait for it to finish, or run 'gbrain doctor' if it has been more than 30 minutes.`,
    );
    // Control: the degraded message carries no holder detail / remediation.
    expect(msg).not.toContain('--break-lock');
    expect(msg).not.toContain(`pid ${process.pid}`);
  });
});