Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
14.2 KB
Raw Blame
import assert from 'node:assert/strict';
import { randomUUID } from 'node:crypto';
import { mkdirSync, writeFileSync } from 'node:fs';
import { dirname, join } from 'node:path';
import { Client } from '@modelcontextprotocol/sdk/client/index.js';
import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js';
import type { BrainEngine } from '../../src/core/engine.ts';
import type { OperationContext } from '../../src/core/ops/contract.ts';
import { claimWorktree } from '../../src/core/persistence/ownership.ts';
import { activateSharedSkillPersistence } from '../../src/core/persistence/skill-activation.ts';
import { declarePersistenceProtocol } from '../../src/core/persistence/protocol.ts';
import { withCoordinatedWrite } from '../../src/core/persistence/context.ts';
import { TEST_WRITE_ATTRIBUTION } from '../../test/helpers/write-attribution.ts';
import { disposePersistenceConsumer } from '../../src/core/persistence/service.ts';
import { sha256 } from '../../src/core/persistence/digest.ts';
import { normalizeSkillFiles, skillMetadata } from '../../src/core/shared-skills/manifest.ts';
import { setSharedSkillPolicy } from '../../src/core/shared-skills/policy.ts';
import { SHARED_SKILL_LIMITS, type SharedSkillDetail, type SharedSkillFileInput } from '../../src/core/shared-skills/model.ts';
import type { MembershipSnapshot, SharedSkillIdentity } from '../../src/core/shared-skills/membership-types.ts';
import { startHttpTransport } from '../../src/mcp/http-transport.ts';
import { RateLimiter } from '../../src/mcp/rate-limit.ts';
import { isolatedSharedSkillsEngine } from '../../test/helpers/shared-skills-engine.ts';
import { LifecycleMeter } from './lifecycle-metrics.ts';

export const LIVE_NAME = 'benchmark-live';
export const PACK_ID = 'benchmark-pack';
export const ASSET_PATH = `skills/${LIVE_NAME}/assets/fixture.bin`;
export const READ_OPS = ['list_skills', 'get_skill', 'get_skill_asset'];
export const MEMBER_OPS = [...READ_OPS, 'join_brain', 'sync_brain_skills', 'leave_brain'];
export const PUBLISHER_OPS = [...READ_OPS, 'put_skill', 'get_write_request'];

export function liveFixture(generation: number, assetBytes: number): { files: SharedSkillFileInput[]; body: string; asset: Buffer; description: string; triggers: string[] } {
  const asset = Buffer.alloc(assetBytes);
  for (let index = 0; index < asset.length; index++) asset[index] = (index * 17 + generation) % 256;
  const description = `Synthetic lifecycle generation ${generation}`;
  const triggers = [`synthetic-generation-${generation}`];
  const body = `---\nname: ${LIVE_NAME}\ndescription: ${description}\ntriggers: [${triggers[0]}]\n---\n\nSynthetic benchmark instructions.\nGeneration: ${generation}\n`;
  return { body, asset, description, triggers, files: [
    { path: `skills/${LIVE_NAME}/SKILL.md`, content: body, file_class: 'prose', depends_on: [ASSET_PATH] },
    { path: ASSET_PATH, content: asset.toString('base64'), encoding: 'base64', file_class: 'asset' },
  ] };
}
export const identity = (skill: SharedSkillIdentity): SharedSkillIdentity => ({ brain_id: skill.brain_id, source_id: skill.source_id,
  source_incarnation: skill.source_incarnation, pack_id: skill.pack_id, name: skill.name, revision: skill.revision });

export interface BenchmarkPeer { client: Client; connection_ms: number; cache: Map<string, string>; joined?: MembershipSnapshot; }
export interface LifecycleFixture {
  engine: BrainEngine;
  raw: BrainEngine;
  incarnation: string;
  meter: LifecycleMeter;
  members: BenchmarkPeer[];
  publisher: BenchmarkPeer;
  seed: { count: number; elapsed_ms: number; canonical_fixture_bytes: number; manifest_bytes: number; mode: string };
  call<T>(peer: BenchmarkPeer, operation: string, params: Record<string, unknown>): Promise<T>;
  verifyAndFetch(peer: BenchmarkPeer, skill: SharedSkillIdentity, latest?: boolean): Promise<SharedSkillDetail>;
  close(): Promise<void>;
}

export async function createLifecycleFixture(options: { root: string; size: number; members: number; assetBytes: number; databaseUrl?: string }): Promise<LifecycleFixture> {
  const isolated = await isolatedSharedSkillsEngine(options.databaseUrl);
  const raw = isolated.engine;
  const meter = new LifecycleMeter();
  const engine = meter.observe(raw);
  let server: Awaited<ReturnType<typeof startHttpTransport>> | undefined;
  const peers: BenchmarkPeer[] = [];
  const close = async () => {
    const clients = await Promise.allSettled(peers.map(peer => peer.client.close()));
    server?.stop(true);
    try { await disposePersistenceConsumer(engine); } finally { await isolated.close(); }
    assert(clients.every(result => result.status === 'fulfilled'), 'Every benchmark client must close cleanly');
  };
  try {
    const started = performance.now();
    const contentRoot = join(options.root, 'content'); mkdirSync(contentRoot, { recursive: true });
    const seeded = Array.from({ length: options.size }, (_, index) => {
      const name = index === 0 ? LIVE_NAME : `fixture-${String(index).padStart(4, '0')}`;
      const input: SharedSkillFileInput[] = index === 0 ? liveFixture(0, options.assetBytes).files : [{ path: `skills/${name}/SKILL.md`,
        content: `---\nname: ${name}\ndescription: Synthetic fixture ${index}\n---\n\nBounded fixture prose ${index}.\n`, file_class: 'prose' }];
      const files = normalizeSkillFiles(name, input);
      return { name, revision: randomUUID(), files, metadata: skillMetadata(name, files, {}) };
    });
    let canonicalBytes = 0;
    for (const row of seeded) for (const file of row.files) {
      const path = join(contentRoot, file.path); const bytes = Buffer.from(file.content, 'base64');
      mkdirSync(dirname(path), { recursive: true }); writeFileSync(path, bytes); canonicalBytes += bytes.length;
    }
    const manifest = { name: PACK_ID, brain_resident: true, version: '1.0.0', skills: seeded.map(row => `skills/${row.name}`),
      shared_skills: { schema_version: 2, skills: seeded.map(row => ({ name: row.name, revision: row.revision,
        files: row.files.map(({ content: _content, ...file }) => file) })) } };
    const manifestContent = `${JSON.stringify(manifest, null, 2)}\n`;
    assert(Buffer.byteLength(manifestContent) + options.assetBytes < SHARED_SKILL_LIMITS.bundleBytes, 'Fixture manifest leaves room inside the real publication byte bound');
    writeFileSync(join(contentRoot, 'skillpack.json'), manifestContent);
    await engine.executeRaw("UPDATE sources SET local_path=$1 WHERE id='default'", [contentRoot]);
    await claimWorktree(engine, 'default', contentRoot);
    await activateSharedSkillPersistence(engine, { confirmQuiesced: true });
    await engine.setConfig('mcp.publish_skills', 'true');
    await engine.setConfig('mcp.strict_params', 'reject');
    const local: OperationContext = { engine, config: { engine: options.databaseUrl ? 'postgres' : 'pglite', embedding_disabled: true },
      sourceId: 'default', remote: false, dryRun: false, logger: { info() {}, warn() {}, error() {} } };
    const policy = await setSharedSkillPolicy(local, 'default', { version: 1, enabled: true, classes: ['prose', 'asset'],
      audiences: ['readers'], requirements: [], allow_follow: true }, null);
    const [source] = await engine.executeRaw<{ incarnation: string }>("SELECT incarnation FROM sources WHERE id='default'");
    await engine.transaction(async tx => {
      await declarePersistenceProtocol(tx);
      await withCoordinatedWrite(tx, ['default'], async () => {
        for (let offset = 0; offset < seeded.length; offset += 100) {
          const rows = JSON.stringify(seeded.slice(offset, offset + 100));
          await tx.executeRaw(`INSERT INTO shared_skill_heads(source_id,source_incarnation,pack_id,name,revision,metadata,policy_epoch)
            SELECT 'default',$1::uuid,$2,r.name,r.revision,r.metadata,$3
            FROM jsonb_to_recordset($4::text::jsonb) AS r(name text,revision uuid,metadata jsonb,files jsonb)`, [source.incarnation, PACK_ID, policy.policy_epoch, rows]);
          await tx.executeRaw(`INSERT INTO shared_skill_revisions(source_id,source_incarnation,pack_id,name,revision,metadata,files,policy_epoch,request_id)
            SELECT 'default',$1::uuid,$2,r.name,r.revision,r.metadata,r.files,$3,$5::uuid
            FROM jsonb_to_recordset($4::text::jsonb) AS r(name text,revision uuid,metadata jsonb,files jsonb)`, [source.incarnation, PACK_ID, policy.policy_epoch, rows, randomUUID()]);
        }
        await tx.executeRaw(`INSERT INTO shared_skill_packs(source_id,source_incarnation,pack_id,revision,manifest,manifest_hash)
          VALUES('default',$1::uuid,$2,$3::uuid,$4::text::jsonb,$5)`, [source.incarnation, PACK_ID, randomUUID(), JSON.stringify(manifest), sha256(manifestContent)]);
      }, TEST_WRITE_ATTRIBUTION);
    });
    const seed = { count: seeded.length, elapsed_ms: performance.now() - started, canonical_fixture_bytes: canonicalBytes,
      manifest_bytes: Buffer.byteLength(manifestContent), mode: 'matching filesystem and sealed projection fixtures seeded directly; NOT a measured canonical publication' };
    server = await startHttpTransport({ port: 0, engine, surface: 'full', limiters: {
      ip: new RateLimiter({ limit: 10_000, windowMs: 60_000, lruCap: 100 }),
      token: new RateLimiter({ limit: 10_000, windowMs: 60_000, lruCap: 100 }),
    } });
    const endpoint = new URL(`http://127.0.0.1:${server.port}/mcp`);
    const credential = async (name: string, scopes: string[], operations: string[]) => {
      const id = randomUUID(); const token = `synthetic-benchmark-${randomUUID()}`;
      await engine.executeRaw(`INSERT INTO access_tokens(id,name,token_hash,scopes,permissions) VALUES($1::uuid,$2,$3,$4::text[],$5::text::jsonb)`,
      [id, name, sha256(token), scopes, JSON.stringify({ source_id: 'default', allowed_operations: operations })]);
      const client = new Client({ name: 'shared-skills-lifecycle-benchmark', version: '1' }, { capabilities: {} });
      const peer: BenchmarkPeer = { client, connection_ms: 0, cache: new Map() }; peers.push(peer);
      const connected = performance.now();
      await client.connect(new StreamableHTTPClientTransport(endpoint, { requestInit: { headers: { Authorization: `Bearer ${token}` } } }),
        { signal: AbortSignal.timeout(30_000) });
      peer.connection_ms = performance.now() - connected;
      return peer;
    };
    const publisher = await credential('synthetic-publisher', ['read', 'write', 'skill_editor'], PUBLISHER_OPS);
    const members = await Promise.all(Array.from({ length: options.members }, (_, index) => credential(`synthetic-member-${index}`,
      ['read', 'skills_member_self'], MEMBER_OPS)));
    async function call<T>(peer: BenchmarkPeer, operation: string, params: Record<string, unknown>): Promise<T> {
      const counters = meter.counters(); counters.tool_calls[operation] = (counters.tool_calls[operation] ?? 0) + 1;
      counters.argument_bytes_by_operation[operation] = (counters.argument_bytes_by_operation[operation] ?? 0) + Buffer.byteLength(JSON.stringify(params));
      try {
        const result = await peer.client.callTool({ name: operation, arguments: params }, undefined, { timeout: 60_000 });
        const resultBytes = Buffer.byteLength(JSON.stringify(result));
        counters.tool_result_json_bytes += resultBytes;
        counters.result_bytes_by_operation[operation] = (counters.result_bytes_by_operation[operation] ?? 0) + resultBytes;
        const text = (result.content as Array<{ type: string; text?: string }>).find(item => item.type === 'text')?.text;
        assert(text, 'Every benchmark operation must return its typed JSON result');
        const payload = JSON.parse(text);
        if (result.isError) {
          if (operation === 'put_skill' && payload.error === 'write_pending' && payload.write_request?.request_id === params.request_id &&
            ['queued', 'running', 'recovering'].includes(payload.write_request.state)) {
            counters.pending_receipts++;
            return payload.write_request as T;
          }
          const error = new Error(`Benchmark operation ${operation} failed`) as Error & { code: unknown; operation: string };
          error.code = payload.error ?? 'operation_failed';
          error.operation = operation;
          throw error;
        }
        return payload as T;
      } catch (error) { counters.errors++; throw error; }
    }
    async function verifyAndFetch(peer: BenchmarkPeer, skill: SharedSkillIdentity, latest = false): Promise<SharedSkillDetail> {
      const { revision, brain_id, ...key } = identity(skill);
      const detail = await call<SharedSkillDetail>(peer, 'get_skill', { ...key, expected_brain_id: brain_id, ...(!latest ? { revision } : {}), schema_version: 2 });
      if (!latest) assert.equal(detail.revision, skill.revision, 'Body fetch must return the exact issued revision');
      assert.equal(detail.delivery, 'complete', 'Benchmark files must have complete approved closure');
      const main = detail.files.find(file => file.path === `skills/${detail.name}/SKILL.md`);
      assert(main); assert.equal(sha256(detail.body), main.sha256, 'Body must match its declared manifest hash');
      for (const file of detail.files.filter(file => file.path !== main.path)) {
        const asset = await call<{ revision: string; content: string; sha256: string; size: number }>(peer, 'get_skill_asset', { ...key, expected_brain_id: brain_id, revision: detail.revision, path: file.path });
        const bytes = Buffer.from(asset.content, 'base64');
        meter.counters().decoded_asset_bytes += bytes.length;
        assert.equal(asset.revision, detail.revision); assert.equal(sha256(bytes), file.sha256); assert.equal(asset.sha256, file.sha256); assert.equal(bytes.length, file.size);
        if (detail.name === LIVE_NAME) {
          const generation = Number(detail.body.match(/^Generation: (\d+)$/m)?.[1]);
          const expected = liveFixture(generation, options.assetBytes);
          assert(Number.isSafeInteger(generation)); assert.deepEqual(bytes, expected.asset);
          assert.equal(detail.body, expected.body); assert.equal(detail.description, expected.description); assert.deepEqual(detail.triggers, expected.triggers);
        }
      }
      peer.cache.set(detail.name, detail.revision);
      return detail;
    }
    return { engine, raw, incarnation: source.incarnation, meter, publisher, members, seed, call, verifyAndFetch, close };
  } catch (error) { await close(); throw error; }
}