Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
8.2 KB
Raw Blame
/**
 * A git effect the effect pass claimed and set aside for its root, followed by a claim that loses its connection (a
 * pooler drop or failover), stayed `running` until its 2-minute claim lease lapsed: the pass threw and nothing gave the
 * claim back. The crash robot's `pooler_disconnect` fault hit this on CI as a ~2-minute stall. The pass now gives back
 * every effect it claimed and had not started, fenced on each claim's token, as soon as the database answers again.
 */
import { describe, expect, test } from 'bun:test';
import { randomUUID } from 'node:crypto';
import type { BrainEngine } from '../src/core/engine.ts';
import { runPersistenceEffects } from '../src/core/persistence/effects.ts';
import { localHostId } from '../src/core/persistence/identity.ts';
import { managedBrain } from './helpers/managed-brain.ts';
import { requirePostgresTestDatabase, testBackends } from './helpers/test-backends.ts';

const connectionLost = () => Object.assign(new Error('write CONNECTION_CLOSED 127.0.0.1:6543'), { code: 'CONNECTION_CLOSED' });

/** The first transactionDirect after a git effect's claim returns fails with a dropped connection. */
function dropAfterGitClaim(engine: BrainEngine): BrainEngine {
  let armed = false; let fired = false;
  return new Proxy(engine, {
    get(target, prop) {
      if (prop === 'transactionDirect') {
        return async (fn: never, opts: never) => {
          if (armed && !fired) { fired = true; throw connectionLost(); }
          const out = await target.transactionDirect(fn, opts) as { kind?: string; state?: string } | null;
          if (out && !Array.isArray(out) && out.kind === 'git' && out.state === 'running') armed = true;
          return out;
        };
      }
      const value = Reflect.get(target, prop, target);
      return typeof value === 'function' ? value.bind(target) : value;
    },
  });
}

async function queueEffects(engine: BrainEngine, git: Record<string, unknown>): Promise<void> {
  const [{ incarnation, worktree }] = await engine.executeRaw<{ incarnation: string; worktree: string }>(
    "SELECT source_incarnation::text AS incarnation, worktree_id::text AS worktree FROM persistence_source_bindings WHERE source_id='default'");
  await engine.transaction(async tx => {
    await tx.executeRaw("SELECT set_config('gbrain.persistence_protocol','2',true)");
    const [request] = await tx.executeRaw<{ id: string }>(`INSERT INTO persistence_requests(principal_kind,principal_id,request_id,operation,source_id,source_incarnation,slug,digest,authority,
        intent_bytes,terminal_reservation,state,completed_at) VALUES ('local_cli','test',$1::uuid,'put_page','default',$2::uuid,'notes/alpha-example','x','{}'::jsonb,0,16384,'committed',now()) RETURNING id::text`,
      [randomUUID(), incarnation]);
    await tx.executeRaw(`INSERT INTO persistence_effects(request_id,kind,data,state,source_id,source_incarnation,worktree_id,next_attempt_at)
      VALUES ($1::uuid,'git',$2::text::jsonb,'queued','default',$3::uuid,$4::uuid,now()-interval '1 minute'),
             ($1::uuid,'links',$5::text::jsonb,'queued','default',$3::uuid,NULL,now())`,
      [request!.id, JSON.stringify(git), incarnation, worktree, JSON.stringify({ slug: 'notes/alpha-example' })]);
  });
}

for (const backend of testBackends()) describe(`[${backend}] an effect pass that loses its connection gives back the git effects it set aside`, () => {
  const databaseUrl = backend === 'postgres' ? requirePostgresTestDatabase() : undefined;
  const cases = [
    ['the next claim', { slug: 'notes/alpha-example', page_id: 1, expected_hash: null, relative_path: 'notes/alpha-example.md', version: 1 }],
    ['the claim of its coalesced siblings', { slug: 'notes/alpha-example', page_id: 1, expected_hash: null, relative_path: 'notes/alpha-example.md' }],
  ] as const;
  for (const [failing, git] of cases) {
    test(`forced probe: ${failing} fails on a dropped connection; the set-aside git effect is queued again, not running under its lease`, () => managedBrain(async ({ engine }) => {
      await queueEffects(engine, git);
      await expect(runPersistenceEffects(dropAfterGitClaim(engine), { engine: engine.kind }, { hostId: localHostId(), limit: 5 }))
        .rejects.toMatchObject({ code: 'CONNECTION_CLOSED' });
      const [effect] = await engine.executeRaw<{ state: string; token: string | null; lease: string | null }>(
        "SELECT state, execution_token::text AS token, claim_expires_at::text AS lease FROM persistence_effects WHERE kind='git'");
      expect(effect).toEqual({ state: 'queued', token: null, lease: null });
    }, { databaseUrl }));
  }

  const flushCases = [['looking up the binding of the effect it just claimed', 0, 2, true], ['flushing the deferred git groups', 2, 1, false]] as const;
  for (const [where, skip, limit, versioned] of flushCases) {
    test(`forced probe: the connection drops while ${where}; every effect claimed and not finished is queued again`, () => managedBrain(async ({ engine }) => {
      for (const n of [1, 2]) await queueEffects(engine, { slug: `notes/alpha-example-${n}`, page_id: n, expected_hash: null, relative_path: `notes/alpha-example-${n}.md`, ...(versioned ? { version: n } : {}) });
      let claims = 0; let passed = 0; let failures = 0;
      const drop = (call: (...a: never[]) => Promise<unknown>) => async (...args: never[]) => {
        if (claims === limit && passed++ >= skip && failures < 2) { failures++; throw connectionLost(); }
        const out = await call(...args) as { kind?: string; state?: string } | null;
        if (out && !Array.isArray(out) && out.kind === 'git' && out.state === 'running') claims++;
        return out;
      };
      const flaky = new Proxy(engine, {
        get(target, prop) {
          const value = Reflect.get(target, prop, target);
          if (prop === 'transactionDirect' || prop === 'executeRaw' || prop === 'transaction') return drop(value.bind(target));
          return typeof value === 'function' ? value.bind(target) : value;
        },
      });
      await expect(runPersistenceEffects(flaky, { engine: engine.kind }, { hostId: localHostId(), limit }))
        .rejects.toMatchObject({ code: 'CONNECTION_CLOSED' });
      expect(failures).toBe(2);
      const states = await engine.executeRaw<{ state: string; token: string | null }>("SELECT state, execution_token::text AS token FROM persistence_effects WHERE kind='git'");
      expect(states).toEqual([{ state: 'queued', token: null }, { state: 'queued', token: null }]);
    }, { databaseUrl }));
  }

  test('forced probe: the give-back stops at once when the worker is aborted, leaving the rest to their lease', () => managedBrain(async ({ engine }) => {
    for (const version of [1, 2, 3]) await queueEffects(engine, { slug: `notes/alpha-example-${version}`, page_id: version, expected_hash: null, relative_path: `notes/alpha-example-${version}.md`, version });
    const abort = new AbortController();
    let claims = 0; let fired = false;
    const flaky = new Proxy(engine, {
      get(target, prop) {
        if (prop === 'transactionDirect') {
          return async (fn: never, opts: never) => {
            if (claims === 3 && !fired) { fired = true; throw connectionLost(); }
            const out = await target.transactionDirect(fn, opts) as { kind?: string; state?: string } | null;
            if (out && !Array.isArray(out) && out.kind === 'git' && out.state === 'running') claims++;
            return out;
          };
        }
        if (prop === 'executeRaw') {
          return async (sql: string, params?: unknown[]) => {
            const out = await target.executeRaw(sql, params);
            if (fired && sql.startsWith("UPDATE persistence_effects SET state='queued'")) abort.abort();
            return out;
          };
        }
        const value = Reflect.get(target, prop, target);
        return typeof value === 'function' ? value.bind(target) : value;
      },
    });
    await expect(runPersistenceEffects(flaky, { engine: engine.kind }, { hostId: localHostId(), limit: 5, signal: abort.signal }))
      .rejects.toMatchObject({ code: 'CONNECTION_CLOSED' });
    const states = await engine.executeRaw<{ state: string }>("SELECT state FROM persistence_effects WHERE kind='git' ORDER BY state");
    expect(states.map(s => s.state)).toEqual(['queued', 'running', 'running']);
  }, { databaseUrl }));
});