Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
14.0 KB
Raw Blame
/**
 * The per-skill benchmark task-set kernel (agent-presence, Lane A).
 *
 * A skill rewrite is gated by measured outcomes, never by vibes: before
 * any candidate instructions are drafted, the skill carries a committed
 * task set - a validated, content-hashed document listing the tasks an
 * agent must perform while following the skill, and the structural
 * predicates the resulting transcript must satisfy. This module is that
 * document's only parser and its only hasher.
 *
 * Posture, pinned here because every later lane leans on it:
 *
 *   - The predicates are STRUCTURAL. A check names a tool, a path shape,
 *     an order, a forbidden step, or an output field class. No check
 *     ever reads prose, and no scorer down-stream of this module ever
 *     will: task text lives in the committed fixture, opaque to the
 *     predicate layer.
 *   - The document is fail-fast: a malformed shape is a named error
 *     carrying the source the document was read from, in the
 *     `parseBenchFixture` style. Nothing partial ever reaches a spec.
 *   - The module is a TRUE LEAF: it imports only the one digest encoding
 *     and the Brain path-name constants, so the import graph cannot grow
 *     a cycle rooted here, and nothing above can be pulled in by
 *     accident - `import type` included, per the architecture ratchet.
 *
 * The task set's home is vault-side, `Brain/skill-benchmarks/<skill>/`,
 * so an operator's benchmarks travel with the vault; the skills root
 * itself may sit outside the vault and cannot carry them. The hash over
 * the canonical task list is what a rewrite spec freezes as its receipt
 * (`skill-rewrite-spec.ts`), so it must be stable across parses and key
 * order and sensitive to any task byte - all three are pinned in tests.
 */

import { posix } from "node:path";

import { canonicalJson, sha256Hex } from "../integrity/digest.ts";
import { BRAIN_ROOT_REL } from "./path-constants.ts";

/**
 * The benchmark directory under `Brain/`, one skill-document folder per
 * skill. A constant, not a literal scattered through callers: the state
 * surface row, the paths builder and the CLI all name it from here.
 */
export const SKILL_BENCHMARKS_DIR = "skill-benchmarks";

/** The vault-relative benchmark root: `Brain/skill-benchmarks`. */
export const SKILL_BENCHMARKS_ROOT_REL = posix.join(BRAIN_ROOT_REL, SKILL_BENCHMARKS_DIR);

/**
 * Compose the vault-relative path of one benchmark artifact for `skill`
 * (the tasks document, the spec receipt, a round's transcript drop site).
 * The name was validated at parse time; composing here keeps the one
 * spelling of the layout in the module that owns it.
 */
export function skillBenchmarkRelPath(skillName: string, ...segments: string[]): string {
  return posix.join(SKILL_BENCHMARKS_ROOT_REL, skillName, ...segments);
}

/** The tasks document inside a skill's benchmark folder. */
export const SKILL_BENCHMARK_TASKS_FILE = "tasks.json";
/** The frozen rewrite spec beside the tasks document. */
export const SKILL_BENCHMARK_SPEC_FILE = "spec.json";

/** The closed task-kind vocabulary. One kind today; the union keeps a
 * second kind from arriving as a string scattered across scorers. */
export const SKILL_BENCH_TASK_KIND = Object.freeze({
  invocation_transcript: "invocation_transcript",
} as const);

export type SkillBenchTaskKind = (typeof SKILL_BENCH_TASK_KIND)[keyof typeof SKILL_BENCH_TASK_KIND];

/** The task-kind members, for parse-time refusal and guard sharing. */
export const SKILL_BENCH_TASK_KINDS: ReadonlyArray<SkillBenchTaskKind> = Object.freeze(
  Object.values(SKILL_BENCH_TASK_KIND),
);

export function isSkillBenchTaskKind(value: unknown): value is SkillBenchTaskKind {
  return (
    typeof value === "string" && (SKILL_BENCH_TASK_KINDS as ReadonlyArray<string>).includes(value)
  );
}

/**
 * The closed check vocabulary - the ONLY predicates a transcript is ever
 * scored against. Structural by construction: a tool name, a path
 * prefix, an order, a forbidden step, an output field class.
 */
export const SKILL_BENCH_CHECK = Object.freeze({
  tool_used: "tool_used",
  artifact_created: "artifact_created",
  step_order: "step_order",
  forbidden_step: "forbidden_step",
  output_field: "output_field",
} as const);

export type SkillBenchCheck = (typeof SKILL_BENCH_CHECK)[keyof typeof SKILL_BENCH_CHECK];

/** The check members, for parse-time refusal and guard sharing. */
export const SKILL_BENCH_CHECKS: ReadonlyArray<SkillBenchCheck> = Object.freeze(
  Object.values(SKILL_BENCH_CHECK),
);

export function isSkillBenchCheck(value: unknown): value is SkillBenchCheck {
  return typeof value === "string" && (SKILL_BENCH_CHECKS as ReadonlyArray<string>).includes(value);
}

/**
 * The closed value classes an `output_field` check can demand. Deliberately
 * small: classes a scorer can decide without ever reading the value's
 * content. `any` is not a member - a check that accepts anything is not a
 * check.
 */
export const SKILL_BENCH_VALUE_CLASS = Object.freeze({
  non_empty_string: "non_empty_string",
  number: "number",
  boolean: "boolean",
  non_empty_array: "non_empty_array",
} as const);

export type SkillBenchValueClass =
  (typeof SKILL_BENCH_VALUE_CLASS)[keyof typeof SKILL_BENCH_VALUE_CLASS];

/** The value-class members, for parse-time refusal and guard sharing. */
export const SKILL_BENCH_VALUE_CLASSES: ReadonlyArray<SkillBenchValueClass> = Object.freeze(
  Object.values(SKILL_BENCH_VALUE_CLASS),
);

export function isSkillBenchValueClass(value: unknown): value is SkillBenchValueClass {
  return (
    typeof value === "string" &&
    (SKILL_BENCH_VALUE_CLASSES as ReadonlyArray<string>).includes(value)
  );
}

/**
 * One structural predicate over the returned transcript. Field spellings
 * match the committed JSON exactly (`path_prefix`, `value_class`) so the
 * parse is a pass-through freeze and the canonical hash over the parsed
 * tasks equals the hash over the document's task list.
 */
export type SkillBenchExpectation =
  | { readonly check: "tool_used"; readonly tool: string }
  | { readonly check: "artifact_created"; readonly path_prefix: string }
  | { readonly check: "step_order"; readonly steps: ReadonlyArray<string> }
  | { readonly check: "forbidden_step"; readonly step: string }
  | {
      readonly check: "output_field";
      readonly field: string;
      readonly value_class: SkillBenchValueClass;
    };

/** One benchmark task: what to do, and what the transcript must show. */
export interface SkillBenchmarkTask {
  /** Filename-safe id; names the transcript drop file for the round. */
  readonly id: string;
  readonly kind: SkillBenchTaskKind;
  /** The task input, carried verbatim into the dev envelope prompt. */
  readonly input: string;
  readonly expects: ReadonlyArray<SkillBenchExpectation>;
}

/** A validated, content-hashed task set for one skill. */
export interface SkillBenchmarkTaskSet {
  readonly skillName: string;
  readonly tasks: ReadonlyArray<SkillBenchmarkTask>;
  /** sha256 over the canonical task list - the spec receipt's anchor. */
  readonly hash: string;
}

/**
 * A skill's benchmark name becomes a DIRECTORY under the benchmark root,
 * so it follows the skill-directory rule: lowercase letters, digits and
 * single hyphens. Spelled here rather than imported from the proposal
 * module because this kernel is a true leaf and must not reach into the
 * layer above; the materialize path re-checks the same rule (and has its
 * own named constant), so a name that passes one but not the other is
 * refused at whichever door it reaches first.
 */
export const SKILL_BENCH_SKILL_NAME_RE = /^[a-z0-9]+(?:-[a-z0-9]+)*$/;

/**
 * Task ids name transcript drop files inside a round directory, so they
 * are filename-safe: letters, digits, dots, hyphens and underscores with
 * a leading alphanumeric. No separators, no traversal.
 */
export const SKILL_BENCH_TASK_ID_RE = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;

/** The canonical content hash over a parsed task list. */
export function skillBenchmarkTaskSetHash(tasks: ReadonlyArray<SkillBenchmarkTask>): string {
  return sha256Hex(canonicalJson(tasks));
}

/**
 * Parse and validate a raw task-set document. Throws named errors that
 * always carry `source`, so an operator sees WHICH document refused.
 */
export function parseSkillBenchmarkTasks(raw: unknown, source: string): SkillBenchmarkTaskSet {
  const at = (what: string): string => `skill benchmark tasks (${source}): ${what}`;
  if (raw === null || typeof raw !== "object" || Array.isArray(raw)) {
    throw new Error(at("expected a JSON object"));
  }
  const record = raw as Record<string, unknown>;

  const skillName = record["skill"];
  if (typeof skillName !== "string" || skillName.trim().length === 0) {
    throw new Error(at("skill must be a non-empty string"));
  }
  if (!SKILL_BENCH_SKILL_NAME_RE.test(skillName)) {
    throw new Error(
      at(
        `skill must be a skill directory name (lowercase letters, digits and single hyphens): ${JSON.stringify(skillName)}`,
      ),
    );
  }

  const tasksRaw = record["tasks"];
  if (!Array.isArray(tasksRaw) || tasksRaw.length === 0) {
    throw new Error(at("tasks must be a non-empty array"));
  }

  const tasks = tasksRaw.map((task, index) => parseTask(task, at, index));
  const seen = new Set<string>();
  for (const task of tasks) {
    if (seen.has(task.id)) {
      throw new Error(at(`duplicate task id: ${task.id}`));
    }
    seen.add(task.id);
  }

  return Object.freeze({
    skillName,
    tasks: Object.freeze(tasks),
    hash: skillBenchmarkTaskSetHash(tasks),
  });
}

function parseTask(raw: unknown, at: (what: string) => string, index: number): SkillBenchmarkTask {
  if (raw === null || typeof raw !== "object" || Array.isArray(raw)) {
    throw new Error(at(`task ${index} must be an object`));
  }
  const record = raw as Record<string, unknown>;

  const id = record["id"];
  if (typeof id !== "string" || id.trim().length === 0) {
    throw new Error(at(`task ${index} needs a non-empty id`));
  }
  if (!SKILL_BENCH_TASK_ID_RE.test(id)) {
    throw new Error(at(`task id must be filename-safe: ${JSON.stringify(id)}`));
  }

  if (!isSkillBenchTaskKind(record["kind"])) {
    throw new Error(
      at(`task ${id} has an unknown kind (expected one of ${SKILL_BENCH_TASK_KINDS.join(", ")})`),
    );
  }

  const input = record["input"];
  if (typeof input !== "string" || input.trim().length === 0) {
    throw new Error(at(`task ${id} needs a non-empty input`));
  }

  const expectsRaw = record["expects"];
  if (!Array.isArray(expectsRaw) || expectsRaw.length === 0) {
    throw new Error(at(`task ${id} expects must be a non-empty array`));
  }
  const expects = expectsRaw.map((expectation) => parseExpectation(expectation, at, id));

  return Object.freeze({ id, kind: record["kind"], input, expects: Object.freeze(expects) });
}

function parseExpectation(
  raw: unknown,
  at: (what: string) => string,
  taskId: string,
): SkillBenchExpectation {
  if (raw === null || typeof raw !== "object" || Array.isArray(raw)) {
    throw new Error(at(`task ${taskId} expectation must be an object`));
  }
  const record = raw as Record<string, unknown>;
  const check = record["check"];
  if (!isSkillBenchCheck(check)) {
    throw new Error(
      at(`task ${taskId} has an unknown check (expected one of ${SKILL_BENCH_CHECKS.join(", ")})`),
    );
  }
  switch (check) {
    case "tool_used": {
      const tool = nonEmptyString(record["tool"]);
      if (tool === undefined) throw new Error(at(`task ${taskId} tool_used needs a tool name`));
      return Object.freeze({ check, tool });
    }
    case "artifact_created": {
      const prefix = nonEmptyString(record["path_prefix"]);
      if (prefix === undefined) {
        throw new Error(at(`task ${taskId} artifact_created needs a path_prefix`));
      }
      if (isAbsoluteLike(prefix)) {
        throw new Error(
          at(`task ${taskId} artifact_created path_prefix must be vault-relative: ${prefix}`),
        );
      }
      return Object.freeze({ check, path_prefix: prefix });
    }
    case "step_order": {
      const stepsRaw = record["steps"];
      if (
        !Array.isArray(stepsRaw) ||
        stepsRaw.length === 0 ||
        !stepsRaw.every((step) => typeof step === "string" && step.trim().length > 0)
      ) {
        throw new Error(at(`task ${taskId} step_order steps must be a non-empty string array`));
      }
      const steps = stepsRaw as string[];
      const seen = new Set<string>();
      for (const step of steps) {
        if (seen.has(step)) {
          throw new Error(at(`task ${taskId} step_order carries a duplicate step: ${step}`));
        }
        seen.add(step);
      }
      return Object.freeze({ check, steps: Object.freeze([...steps]) });
    }
    case "forbidden_step": {
      const step = nonEmptyString(record["step"]);
      if (step === undefined) {
        throw new Error(at(`task ${taskId} forbidden_step needs a step name`));
      }
      return Object.freeze({ check, step });
    }
    case "output_field": {
      const field = nonEmptyString(record["field"]);
      if (field === undefined) {
        throw new Error(at(`task ${taskId} output_field needs a field name`));
      }
      const valueClass = record["value_class"];
      if (!isSkillBenchValueClass(valueClass)) {
        throw new Error(
          at(
            `task ${taskId} output_field has an unknown value_class (expected one of ${SKILL_BENCH_VALUE_CLASSES.join(", ")})`,
          ),
        );
      }
      return Object.freeze({ check, field, value_class: valueClass });
    }
  }
}

/** Non-empty-string reader used by the per-check parsers. */
function nonEmptyString(value: unknown): string | undefined {
  return typeof value === "string" && value.trim().length > 0 ? value.trim() : undefined;
}

/**
 * Vault-relative shape check for an `artifact_created` prefix: no
 * absolute form, no traversal segment, on either separator - the prefix
 * is matched against vault-relative artifact paths, so an absolute or
 * escaping prefix could never match and only ever misleads.
 */
function isAbsoluteLike(prefix: string): boolean {
  const normalized = prefix.replaceAll("\\", "/");
  return posix.isAbsolute(normalized) || normalized.split("/").includes("..");
}