Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
13.9 KB
Raw Blame
/**
 * The per-skill benchmark task-set kernel (agent-presence, Lane A, A1).
 *
 * The parser is the only door to a task set: every malformed shape is a
 * named error carrying the source the document was read from, the
 * vocabularies are frozen and answered by their guards, and the hash is
 * the one digest encoding over the canonical task list - stable across
 * parses and key order, sensitive to any task byte, because a spec
 * freezes that hash as its receipt.
 */

import { describe, expect, test } from "bun:test";
import { readFileSync } from "node:fs";
import { join } from "node:path";

import {
  SKILL_BENCHMARKS_DIR,
  SKILL_BENCHMARKS_ROOT_REL,
  SKILL_BENCH_CHECK,
  SKILL_BENCH_CHECKS,
  SKILL_BENCH_TASK_ID_RE,
  SKILL_BENCH_TASK_KIND,
  SKILL_BENCH_TASK_KINDS,
  SKILL_BENCH_VALUE_CLASS,
  SKILL_BENCH_VALUE_CLASSES,
  isSkillBenchCheck,
  isSkillBenchTaskKind,
  isSkillBenchValueClass,
  parseSkillBenchmarkTasks,
  skillBenchmarkRelPath,
  skillBenchmarkTaskSetHash,
} from "../../../src/core/brain/skill-rewrite-tasks.ts";

const REPO_ROOT = join(import.meta.dir, "..", "..", "..");
const FIXTURE_PATH = join(REPO_ROOT, "tests", "fixtures", "skill-bench", "example-skill");

function loadFixture(name: string): unknown {
  return JSON.parse(readFileSync(join(FIXTURE_PATH, name), "utf8"));
}

describe("skill-benchmark path constants", () => {
  test("the benchmark root composes from the Brain root and the directory name", () => {
    expect(SKILL_BENCHMARKS_DIR).toBe("skill-benchmarks");
    expect(SKILL_BENCHMARKS_ROOT_REL).toBe("Brain/skill-benchmarks");
    expect(skillBenchmarkRelPath("example-skill", "tasks.json")).toBe(
      "Brain/skill-benchmarks/example-skill/tasks.json",
    );
  });
});

describe("closed vocabularies", () => {
  test("task kinds are frozen and answered by their guard", () => {
    expect(Object.isFrozen(SKILL_BENCH_TASK_KIND)).toBe(true);
    expect(Object.isFrozen(SKILL_BENCH_TASK_KINDS)).toBe(true);
    expect([...SKILL_BENCH_TASK_KINDS]).toEqual(["invocation_transcript"]);
    expect(isSkillBenchTaskKind("invocation_transcript")).toBe(true);
    expect(isSkillBenchTaskKind("freeform_chat")).toBe(false);
    expect(isSkillBenchTaskKind(42)).toBe(false);
  });

  test("checks are frozen and answered by their guard", () => {
    expect(Object.isFrozen(SKILL_BENCH_CHECK)).toBe(true);
    expect(Object.isFrozen(SKILL_BENCH_CHECKS)).toBe(true);
    expect([...SKILL_BENCH_CHECKS]).toEqual([
      "tool_used",
      "artifact_created",
      "step_order",
      "forbidden_step",
      "output_field",
    ]);
    for (const check of SKILL_BENCH_CHECKS) expect(isSkillBenchCheck(check)).toBe(true);
    expect(isSkillBenchCheck("vibe_score")).toBe(false);
    expect(isSkillBenchCheck(undefined)).toBe(false);
  });

  test("output value classes are frozen and answered by their guard", () => {
    expect(Object.isFrozen(SKILL_BENCH_VALUE_CLASS)).toBe(true);
    expect(Object.isFrozen(SKILL_BENCH_VALUE_CLASSES)).toBe(true);
    expect([...SKILL_BENCH_VALUE_CLASSES].toSorted()).toEqual([
      "boolean",
      "non_empty_array",
      "non_empty_string",
      "number",
    ]);
    for (const valueClass of SKILL_BENCH_VALUE_CLASSES) {
      expect(isSkillBenchValueClass(valueClass)).toBe(true);
    }
    expect(isSkillBenchValueClass("any")).toBe(false);
  });

  test("task ids stay filename-safe", () => {
    expect(SKILL_BENCH_TASK_ID_RE.test("t-01")).toBe(true);
    expect(SKILL_BENCH_TASK_ID_RE.test("step_2.json")).toBe(true);
    expect(SKILL_BENCH_TASK_ID_RE.test("../escape")).toBe(false);
    expect(SKILL_BENCH_TASK_ID_RE.test("has space")).toBe(false);
  });
});

describe("parseSkillBenchmarkTasks", () => {
  test("accepts the committed example fixture", () => {
    const set = parseSkillBenchmarkTasks(loadFixture("tasks.json"), "tasks.json");
    expect(set.skillName).toBe("example-skill");
    expect(set.tasks.length).toBe(10);
    expect(set.tasks[0]?.kind).toBe("invocation_transcript");
    expect(set.tasks[0]?.expects[0]).toEqual({ check: "tool_used", tool: "note_write" });
    // The parsed set is frozen at every level, so a caller cannot mutate
    // a receipt after the hash was taken over it.
    expect(Object.isFrozen(set)).toBe(true);
    expect(Object.isFrozen(set.tasks)).toBe(true);
    expect(Object.isFrozen(set.tasks[0]?.expects)).toBe(true);
  });

  test("every parsed task id is filename-safe", () => {
    const set = parseSkillBenchmarkTasks(loadFixture("tasks.json"), "tasks.json");
    for (const task of set.tasks) {
      expect(SKILL_BENCH_TASK_ID_RE.test(task.id)).toBe(true);
    }
  });

  test("the hash is 64 hex characters", () => {
    const set = parseSkillBenchmarkTasks(loadFixture("tasks.json"), "tasks.json");
    expect(set.hash).toMatch(/^[0-9a-f]{64}$/);
  });

  test("the hash is stable across parses and key order", () => {
    const raw = loadFixture("tasks.json") as Record<string, unknown>;
    const first = parseSkillBenchmarkTasks(raw, "a");
    const second = parseSkillBenchmarkTasks(raw, "b");
    expect(first.hash).toBe(second.hash);
    // The same tasks under a different OBJECT key order hash identically:
    // the digest is taken over the canonical form, never the file bytes.
    // (Array order is list meaning and stays significant.)
    const tasks = raw["tasks"] as ReadonlyArray<Record<string, unknown>>;
    const reordered = {
      tasks: tasks.map((task) => {
        const entries = Object.entries(task).toSorted(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0));
        return Object.fromEntries(entries);
      }),
      skill: raw["skill"],
    };
    expect(parseSkillBenchmarkTasks(reordered, "c").hash).toBe(first.hash);
  });

  test("the hash moves when any single task moves", () => {
    const raw = loadFixture("tasks.json") as {
      skill: string;
      tasks: Array<Record<string, unknown>>;
    };
    const baseline = parseSkillBenchmarkTasks(raw, "baseline").hash;
    const unmoved: string[] = [];
    for (let index = 0; index < raw.tasks.length; index++) {
      const touched = structuredClone(raw);
      const task = touched.tasks[index] as { input: string };
      task.input = `${task.input}x`;
      const hash = parseSkillBenchmarkTasks(touched, `touched-${index}`).hash;
      if (hash === baseline) unmoved.push(touched.tasks[index]!.id as string);
    }
    expect(unmoved).toEqual([]);
  });

  test("the hash moves when an expectation moves", () => {
    const raw = loadFixture("tasks.json") as {
      skill: string;
      tasks: Array<{ expects: Array<Record<string, unknown>> }>;
    };
    const baseline = parseSkillBenchmarkTasks(raw, "baseline").hash;
    const touched = structuredClone(raw);
    const expectation = touched.tasks[0]!.expects[0] as { tool: string };
    expectation.tool = "other_tool";
    expect(parseSkillBenchmarkTasks(touched, "touched").hash).not.toBe(baseline);
  });

  test("the task-set hash helper agrees with the parse", () => {
    const set = parseSkillBenchmarkTasks(loadFixture("tasks.json"), "tasks.json");
    expect(skillBenchmarkTaskSetHash(set.tasks)).toBe(set.hash);
  });

  test("refuses every malformed shape by name, naming the source", () => {
    const source = "Brain/skill-benchmarks/example-skill/tasks.json";
    const refusing = (raw: unknown, fragment: string): void => {
      try {
        parseSkillBenchmarkTasks(raw, source);
      } catch (error) {
        expect((error as Error).message).toContain(source);
        expect((error as Error).message).toContain(fragment);
        return;
      }
      throw new Error(`expected a refusal containing "${fragment}"`);
    };

    refusing(null, "expected a JSON object");
    refusing("tasks", "expected a JSON object");
    refusing([], "expected a JSON object");
    refusing({}, "skill");
    refusing({ skill: "example-skill" }, "tasks must be a non-empty array");
    refusing({ skill: "example-skill", tasks: [] }, "tasks must be a non-empty array");
    refusing({ tasks: [] }, "skill");
    refusing(
      {
        skill: "Example Skill",
        tasks: [
          {
            id: "t",
            kind: "invocation_transcript",
            input: "x",
            expects: [{ check: "tool_used", tool: "y" }],
          },
        ],
      },
      "skill directory name",
    );
    refusing(
      {
        skill: "../escape",
        tasks: [
          {
            id: "t",
            kind: "invocation_transcript",
            input: "x",
            expects: [{ check: "tool_used", tool: "y" }],
          },
        ],
      },
      "skill directory name",
    );
    refusing({ skill: "example-skill", tasks: [{}] }, "needs a non-empty id");
    refusing({ skill: "example-skill", tasks: [{ id: "" }] }, "needs a non-empty id");
    refusing(
      {
        skill: "example-skill",
        tasks: [
          {
            id: "t-01",
            kind: "freeform_chat",
            input: "x",
            expects: [{ check: "tool_used", tool: "y" }],
          },
        ],
      },
      "unknown kind",
    );
    refusing(
      {
        skill: "example-skill",
        tasks: [
          {
            id: "t-01",
            kind: "invocation_transcript",
            expects: [{ check: "tool_used", tool: "y" }],
          },
        ],
      },
      "input",
    );
    refusing(
      {
        skill: "example-skill",
        tasks: [
          {
            id: "t-01",
            kind: "invocation_transcript",
            input: 7,
            expects: [{ check: "tool_used", tool: "y" }],
          },
        ],
      },
      "input",
    );
    refusing(
      {
        skill: "example-skill",
        tasks: [{ id: "t-01", kind: "invocation_transcript", input: "x" }],
      },
      "expects must be a non-empty array",
    );
    refusing(
      {
        skill: "example-skill",
        tasks: [{ id: "t-01", kind: "invocation_transcript", input: "x", expects: [] }],
      },
      "expects must be a non-empty array",
    );
    refusing(
      {
        skill: "example-skill",
        tasks: [{ id: "t-01", kind: "invocation_transcript", input: "x", expects: [{}] }],
      },
      "unknown check",
    );
    refusing(
      {
        skill: "example-skill",
        tasks: [
          {
            id: "t-01",
            kind: "invocation_transcript",
            input: "x",
            expects: [{ check: "vibe_score" }],
          },
        ],
      },
      "unknown check",
    );
    refusing(
      {
        skill: "example-skill",
        tasks: [
          {
            id: "t-01",
            kind: "invocation_transcript",
            input: "x",
            expects: [{ check: "tool_used" }],
          },
        ],
      },
      "tool",
    );
    refusing(
      {
        skill: "example-skill",
        tasks: [
          {
            id: "t-01",
            kind: "invocation_transcript",
            input: "x",
            expects: [{ check: "artifact_created", path_prefix: "/abs" }],
          },
        ],
      },
      "path_prefix",
    );
    refusing(
      {
        skill: "example-skill",
        tasks: [
          {
            id: "t-01",
            kind: "invocation_transcript",
            input: "x",
            expects: [{ check: "artifact_created", path_prefix: "a/../b" }],
          },
        ],
      },
      "path_prefix",
    );
    refusing(
      {
        skill: "example-skill",
        tasks: [
          {
            id: "t-01",
            kind: "invocation_transcript",
            input: "x",
            expects: [{ check: "step_order", steps: [] }],
          },
        ],
      },
      "steps must be a non-empty string array",
    );
    refusing(
      {
        skill: "example-skill",
        tasks: [
          {
            id: "t-01",
            kind: "invocation_transcript",
            input: "x",
            expects: [{ check: "step_order", steps: ["a", "a"] }],
          },
        ],
      },
      "duplicate step",
    );
    refusing(
      {
        skill: "example-skill",
        tasks: [
          {
            id: "t-01",
            kind: "invocation_transcript",
            input: "x",
            expects: [{ check: "forbidden_step" }],
          },
        ],
      },
      "step",
    );
    refusing(
      {
        skill: "example-skill",
        tasks: [
          {
            id: "t-01",
            kind: "invocation_transcript",
            input: "x",
            expects: [{ check: "output_field", field: "f" }],
          },
        ],
      },
      "value_class",
    );
    refusing(
      {
        skill: "example-skill",
        tasks: [
          {
            id: "t-01",
            kind: "invocation_transcript",
            input: "x",
            expects: [{ check: "output_field", field: "f", value_class: "any" }],
          },
        ],
      },
      "value_class",
    );
    refusing(
      {
        skill: "example-skill",
        tasks: [
          {
            id: "t-01",
            kind: "invocation_transcript",
            input: "x",
            expects: [{ check: "output_field", value_class: "number" }],
          },
        ],
      },
      "field",
    );
    refusing(
      {
        skill: "example-skill",
        tasks: [
          {
            id: "t-01",
            kind: "invocation_transcript",
            input: "x",
            expects: [{ check: "tool_used", tool: "y" }],
          },
          {
            id: "t-01",
            kind: "invocation_transcript",
            input: "z",
            expects: [{ check: "tool_used", tool: "y" }],
          },
        ],
      },
      "duplicate task id",
    );
  });
});

describe("the kernel stays a true leaf", () => {
  test("it imports nothing beyond the digest and path-name leaves", () => {
    const text = readFileSync(
      join(REPO_ROOT, "src", "core", "brain", "skill-rewrite-tasks.ts"),
      "utf8",
    );
    const imports = [...text.matchAll(/^import\s[^;]*?from\s*"([^"]+)"/gm)].map((m) => m[1]!);
    const allowed = new Set(["../integrity/digest.ts", "./path-constants.ts", "node:path"]);
    const unexpected = imports.filter((specifier) => !allowed.has(specifier));
    expect(unexpected).toEqual([]);
  });
});