Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
28.6 KB
Raw Blame
/**
 * The rewrite gate kernels (agent-presence, Lane A, A3).
 *
 * OSB never calls a model. The envelope builder hands the calling agent
 * one step per DEV task - held-out ids are refused by construction and
 * their task text never reaches a prompt before the single held-out
 * check. The scorer reads structural predicates only and refuses any
 * submission carrying a self-reported success field, so nothing the
 * agent claims about its own work can move the metric. Selection is the
 * preregistered three arms over the receipt's FULL dev membership (a
 * self-selected subset is not enough evidence, and an unregistered id
 * is a refusal), the held-out gate runs once on the full held-out
 * membership and downgrades a promoted candidate on a miss, and every
 * transcript is bound to the candidate body it was produced against.
 * The grade record carries ids, scores and check verdicts with the spec
 * hash - never task text, never transcript text.
 */

import { describe, expect, test } from "bun:test";
import { readFileSync } from "node:fs";
import { join } from "node:path";

import { NEEDS_LLM_STEP } from "../../../src/core/brain/llm-step.ts";
import {
  SKILL_BENCH_CHECK,
  type SkillBenchmarkTask,
} from "../../../src/core/brain/skill-rewrite-tasks.ts";
import {
  SKILL_REWRITE_ROUND_ARM,
  SKILL_REWRITE_STEP,
  SKILL_REWRITE_VERDICT,
  buildRewriteEnvelopes,
  buildRewriteGradeRecord,
  buildRewriteHeldoutEnvelopes,
  checkRewriteHeldout,
  resolveSkillRewriteMode,
  runRewriteJudge,
  scoreRewriteTranscripts,
  selectRewriteCandidate,
  type RewriteSubmission,
  type RewriteTaskScore,
} from "../../../src/core/brain/skill-rewrite.ts";
import {
  buildSkillRewriteSpec,
  SKILL_REWRITE_METRIC,
  type SkillRewriteSpec,
} from "../../../src/core/brain/skill-rewrite-spec.ts";
import { parseSkillBenchmarkTasks } from "../../../src/core/brain/skill-rewrite-tasks.ts";
import { sha256Hex } from "../../../src/core/integrity/digest.ts";

const REPO_ROOT = join(import.meta.dir, "..", "..", "..");
const FIXTURE_PATH = join(REPO_ROOT, "tests", "fixtures", "skill-bench", "example-skill");

const TASKS = parseSkillBenchmarkTasks(
  JSON.parse(readFileSync(join(FIXTURE_PATH, "tasks.json"), "utf8")),
  "tasks.json",
);
const SPEC = buildSkillRewriteSpec(TASKS.skillName, TASKS);
const CANDIDATE_BODY =
  "# example-skill\n\nRewritten instructions: search first, then write, and report the path.";

/** One structurally passing transcript for a task. */
function passingSubmission(task: SkillBenchmarkTask): RewriteSubmission {
  const expectations = task.expects.map((expectation) => {
    switch (expectation.check) {
      case "tool_used":
        return { tools: [expectation.tool] };
      case "artifact_created":
        return { artifacts: [expectation.path_prefix + "result.md"] };
      case "step_order":
        return { steps: [...expectation.steps] };
      case "forbidden_step":
        return { steps: ["search"] };
      case "output_field":
        return {
          output:
            expectation.value_class === "non_empty_string"
              ? { [expectation.field]: "recorded" }
              : expectation.value_class === "number"
                ? { [expectation.field]: 3 }
                : expectation.value_class === "boolean"
                  ? { [expectation.field]: true }
                  : { [expectation.field]: ["a"] },
        };
    }
  });
  return {
    task_id: task.id,
    steps: expectations.flatMap((entry) => entry.steps ?? ["search"]),
    tools_used: expectations.flatMap((entry) => entry.tools ?? []),
    artifacts: expectations.flatMap((entry) => entry.artifacts ?? []),
    output: Object.assign({}, ...expectations.map((entry) => entry.output ?? {})),
  };
}

function failingSubmission(task: SkillBenchmarkTask): RewriteSubmission {
  return { task_id: task.id, steps: [], tools_used: [], artifacts: [], output: {} };
}

describe("buildRewriteEnvelopes", () => {
  test("emits one needs-llm-step per DEV task carrying the body and the task input", () => {
    const envelopes = buildRewriteEnvelopes(CANDIDATE_BODY, SPEC, TASKS);
    expect(envelopes.length).toBe(SPEC.devTaskIds.length);
    for (const envelope of envelopes) {
      expect(envelope.status).toBe(NEEDS_LLM_STEP);
      expect(envelope.step).toBe(SKILL_REWRITE_STEP);
      expect(SPEC.devTaskIds).toContain(String(envelope["task_id"]));
      expect(String(envelope["round"])).toBe(SKILL_REWRITE_ROUND_ARM.candidate);
      expect(envelope.prompt).toContain(CANDIDATE_BODY);
    }
    const inputs = TASKS.tasks.filter((task) => SPEC.devTaskIds.includes(task.id));
    for (const task of inputs) {
      const envelope = envelopes.find((candidate) => candidate["task_id"] === task.id);
      expect(envelope?.prompt).toContain(task.input);
    }
  });

  test("target paths drop into the skill's benchmark rounds tree", () => {
    const envelopes = buildRewriteEnvelopes(CANDIDATE_BODY, SPEC, TASKS, {
      round: SKILL_REWRITE_ROUND_ARM.baseline,
    });
    for (const envelope of envelopes) {
      expect(envelope.target_path).toBe(
        `Brain/skill-benchmarks/example-skill/rounds/baseline/${String(envelope["task_id"])}.json`,
      );
      expect(String(envelope["round"])).toBe(SKILL_REWRITE_ROUND_ARM.baseline);
    }
  });

  test("envelopes carry the candidate body hash the transcript must echo", () => {
    const envelopes = buildRewriteEnvelopes(CANDIDATE_BODY, SPEC, TASKS);
    const heldout = buildRewriteHeldoutEnvelopes(CANDIDATE_BODY, SPEC, TASKS);
    const expected = sha256Hex(CANDIDATE_BODY);
    for (const envelope of [...envelopes, ...heldout]) {
      expect(envelope.candidate_sha256).toBe(expected);
      // The prompt names the binding too: the executing agent reads the
      // prompt, not the envelope fields.
      expect(envelope.prompt).toContain(expected);
    }
  });

  test("the baseline round never names the candidate body a held-out task", () => {
    const envelopes = buildRewriteEnvelopes(CANDIDATE_BODY, SPEC, TASKS);
    const heldout = TASKS.tasks.filter((task) => SPEC.heldoutTaskIds.includes(task.id));
    expect(heldout.length).toBeGreaterThan(0);
    for (const envelope of envelopes) {
      expect(SPEC.heldoutTaskIds).not.toContain(String(envelope["task_id"]));
      for (const task of heldout) {
        expect(envelope.prompt).not.toContain(task.input);
      }
    }
  });

  test("refuses by name when the task set cannot cover the dev receipt", () => {
    const spec: SkillRewriteSpec = {
      ...SPEC,
      devTaskIds: [...SPEC.devTaskIds, "phantom-task"],
    };
    expect(() => buildRewriteEnvelopes(CANDIDATE_BODY, spec, TASKS)).toThrow(/phantom-task/);
  });

  test("refuses by name when the task set hash no longer matches the receipt", () => {
    expect(() => buildRewriteEnvelopes(CANDIDATE_BODY, SPEC, TASKS)).not.toThrow();
    const staleSpec: SkillRewriteSpec = { ...SPEC, tasksHash: `${SPEC.tasksHash.slice(0, 8)}dead` };
    expect(() => buildRewriteEnvelopes(CANDIDATE_BODY, staleSpec, TASKS)).toThrow(/hash/);
  });
});

describe("buildRewriteHeldoutEnvelopes", () => {
  test("emits envelopes for the held-out members only, after promotion", () => {
    const envelopes = buildRewriteHeldoutEnvelopes(CANDIDATE_BODY, SPEC, TASKS);
    expect(envelopes.length).toBe(SPEC.heldoutTaskIds.length);
    for (const envelope of envelopes) {
      expect(SPEC.heldoutTaskIds).toContain(String(envelope["task_id"]));
      expect(String(envelope["round"])).toBe(SKILL_REWRITE_ROUND_ARM.heldout);
      expect(envelope.prompt).toContain(CANDIDATE_BODY);
    }
    const heldoutInputs = TASKS.tasks.filter((task) => SPEC.heldoutTaskIds.includes(task.id));
    for (const task of heldoutInputs) {
      const envelope = envelopes.find((candidate) => candidate["task_id"] === task.id);
      expect(envelope?.prompt).toContain(task.input);
    }
  });
});

describe("scoreRewriteTranscripts", () => {
  test("scores structural predicates and reports failures by check id", () => {
    const devTasks = TASKS.tasks.filter((task) => SPEC.devTaskIds.includes(task.id));
    const passing = devTasks.map(passingSubmission);
    const scores = scoreRewriteTranscripts(passing, TASKS, { arm: "dev", spec: SPEC });
    expect(scores.length).toBe(devTasks.length);
    for (const score of scores) expect(score.passed).toBe(true);

    const failing = devTasks.map(failingSubmission);
    const failedScores = scoreRewriteTranscripts(failing, TASKS, { arm: "dev", spec: SPEC });
    for (const score of failedScores) {
      expect(score.passed).toBe(false);
      // Every check an empty transcript can fail shows up by check id.
      // The exception is forbidden_step, which an empty transcript
      // satisfies: the step was not taken.
      const task = TASKS.tasks.find((candidate) => candidate.id === score.taskId)!;
      const expectedChecks = task.expects.map((expectation) => expectation.check);
      for (const check of expectedChecks) {
        if (check === SKILL_BENCH_CHECK.forbidden_step) {
          expect(score.failedChecks).not.toContain(check);
        } else {
          expect(score.failedChecks).toContain(check);
        }
      }
    }
  });

  test("a partial pass reports exactly the checks that failed", () => {
    const toolCheckTask = TASKS.tasks.find(
      (task) =>
        SPEC.devTaskIds.includes(task.id) &&
        task.expects.some((expectation) => expectation.check === "tool_used"),
    );
    if (toolCheckTask === undefined) throw new Error("fixture has no tool_used task");
    const submission = passingSubmission(toolCheckTask);
    const score = scoreRewriteTranscripts([submission], TASKS, { arm: "dev", spec: SPEC })[0]!;
    expect(score.passed).toBe(true);

    const withoutTool = { ...submission, tools_used: [] };
    const broken = scoreRewriteTranscripts([withoutTool], TASKS, { arm: "dev", spec: SPEC })[0]!;
    expect(broken.passed).toBe(false);
    expect(broken.failedChecks).toEqual([SKILL_BENCH_CHECK.tool_used]);
  });

  test("step_order passes on a subsequence, not only an exact list", () => {
    const orderTask = TASKS.tasks.find(
      (task) =>
        SPEC.devTaskIds.includes(task.id) &&
        task.expects.some((expectation) => expectation.check === "step_order"),
    );
    if (orderTask === undefined) throw new Error("fixture has no step_order task");
    const expectation = orderTask.expects.find((candidate) => candidate.check === "step_order") as {
      steps: ReadonlyArray<string>;
    };
    const withExtraSteps: RewriteSubmission = {
      task_id: orderTask.id,
      steps: ["plan", ...expectation.steps, "verify"],
      tools_used: [],
      artifacts: [],
      output: {},
    };
    // Only the step_order check is guaranteed here; other checks on the
    // same task may fail, so assert on the one predicate under test.
    const score = scoreRewriteTranscripts([withExtraSteps], TASKS, { arm: "dev", spec: SPEC })[0]!;
    expect(score.failedChecks).not.toContain(SKILL_BENCH_CHECK.step_order);
  });

  test("refuses malformed submissions by name", () => {
    const refusing = (submission: unknown, fragment: string): void => {
      try {
        scoreRewriteTranscripts([submission], TASKS);
      } catch (error) {
        expect((error as Error).message).toContain(fragment);
        return;
      }
      throw new Error(`expected a refusal containing "${fragment}"`);
    };
    refusing(null, "submission must be an object");
    refusing({ steps: [] }, "task_id");
    refusing({ task_id: "ghost-task" }, "unknown task id");
    refusing({ task_id: TASKS.tasks[0]!.id, steps: "read" }, "steps must be a string array");
    refusing({ task_id: TASKS.tasks[0]!.id, tools_used: [7] }, "tools_used must be a string array");
    refusing({ task_id: TASKS.tasks[0]!.id, artifacts: [{}] }, "artifacts must be a string array");
    refusing({ task_id: TASKS.tasks[0]!.id, output: "done" }, "output must be an object");
    refusing({ task_id: TASKS.tasks[0]!.id, success: true }, "self-reported");
    refusing({ task_id: TASKS.tasks[0]!.id, passed: true }, "self-reported");
    refusing({ task_id: TASKS.tasks[0]!.id, verdict: true }, "unexpected key");
  });

  test("refuses a duplicate submission for the same task", () => {
    const submission = passingSubmission(TASKS.tasks[0]!);
    expect(() => scoreRewriteTranscripts([submission, submission], TASKS)).toThrow(/twice/);
  });

  test("the dev arm refuses held-out submissions and the held-out arm refuses dev ones", () => {
    const heldoutTask = TASKS.tasks.find((task) => SPEC.heldoutTaskIds.includes(task.id))!;
    const devTask = TASKS.tasks.find((task) => SPEC.devTaskIds.includes(task.id))!;
    expect(() =>
      scoreRewriteTranscripts([passingSubmission(heldoutTask)], TASKS, { arm: "dev", spec: SPEC }),
    ).toThrow(/held-out/);
    expect(() =>
      scoreRewriteTranscripts([passingSubmission(devTask)], TASKS, { arm: "heldout", spec: SPEC }),
    ).toThrow(/dev/);
  });

  test("with the current candidate hash required, a bound transcript scores and stale or missing bindings refuse", () => {
    const devTask = TASKS.tasks.find((task) => SPEC.devTaskIds.includes(task.id))!;
    const candidateHash = sha256Hex(CANDIDATE_BODY);
    const bound: RewriteSubmission = {
      ...passingSubmission(devTask),
      candidate_sha256: candidateHash,
    };
    const scores = scoreRewriteTranscripts([bound], TASKS, {
      arm: "dev",
      spec: SPEC,
      candidateHash,
    });
    expect(scores[0]?.passed).toBe(true);

    // A transcript left over from a previous candidate body never gates
    // this one.
    const stale: RewriteSubmission = {
      ...passingSubmission(devTask),
      candidate_sha256: "0".repeat(64),
    };
    expect(() =>
      scoreRewriteTranscripts([stale], TASKS, { arm: "dev", spec: SPEC, candidateHash }),
    ).toThrow(/does not match the current candidate body/);

    const unbound = passingSubmission(devTask);
    expect(() =>
      scoreRewriteTranscripts([unbound], TASKS, { arm: "dev", spec: SPEC, candidateHash }),
    ).toThrow(/candidate_sha256/);
  });

  test("candidate_sha256 is shape-checked even when no binding is enforced", () => {
    const devTask = TASKS.tasks.find((task) => SPEC.devTaskIds.includes(task.id))!;
    const malformed: RewriteSubmission = { ...passingSubmission(devTask), candidate_sha256: "zz" };
    expect(() => scoreRewriteTranscripts([malformed], TASKS)).toThrow(/candidate_sha256/);
    const wellFormed: RewriteSubmission = {
      ...passingSubmission(devTask),
      candidate_sha256: sha256Hex(CANDIDATE_BODY),
    };
    expect(() => scoreRewriteTranscripts([wellFormed], TASKS)).not.toThrow();
  });

  test("scores come back sorted by task id", () => {
    const devTasks = TASKS.tasks.filter((task) => SPEC.devTaskIds.includes(task.id));
    const submissions = devTasks.map(passingSubmission).toSorted(() => -1);
    const scores = scoreRewriteTranscripts(submissions, TASKS, { arm: "dev", spec: SPEC });
    const ids = scores.map((score) => score.taskId);
    expect(ids).toEqual([...ids].toSorted());
  });
});

describe("selectRewriteCandidate", () => {
  const devTasks = TASKS.tasks.filter((task) => SPEC.devTaskIds.includes(task.id));

  test("a strict beat beyond the margin with n met promotes", () => {
    const baseline = scoreRewriteTranscripts(devTasks.map(failingSubmission), TASKS, {
      arm: "dev",
      spec: SPEC,
    });
    const candidate = scoreRewriteTranscripts(devTasks.map(passingSubmission), TASKS, {
      arm: "dev",
      spec: SPEC,
    });
    const outcome = selectRewriteCandidate(baseline, candidate, SPEC);
    expect(outcome.verdict).toBe(SKILL_REWRITE_VERDICT.promote);
    expect(outcome.candidateRate).toBeGreaterThan(outcome.baselineRate);
  });

  test("a tie holds", () => {
    const baseline = scoreRewriteTranscripts(devTasks.map(passingSubmission), TASKS, {
      arm: "dev",
      spec: SPEC,
    });
    const candidate = scoreRewriteTranscripts(devTasks.map(passingSubmission), TASKS, {
      arm: "dev",
      spec: SPEC,
    });
    const outcome = selectRewriteCandidate(baseline, candidate, SPEC);
    expect(outcome.verdict).toBe(SKILL_REWRITE_VERDICT.hold);
    expect(outcome.baselineRate).toBe(outcome.candidateRate);
  });

  test("a beat under the margin holds", () => {
    const baseline = scoreRewriteTranscripts(devTasks.map(passingSubmission), TASKS, {
      arm: "dev",
      spec: SPEC,
    });
    // Drop exactly one pass: a small rate loss that must stay under the
    // margin for a task set this size.
    const partial = devTasks.map((task, index) =>
      index === 0 ? failingSubmission(task) : passingSubmission(task),
    );
    const candidate = scoreRewriteTranscripts(partial, TASKS, { arm: "dev", spec: SPEC });
    const outcome = selectRewriteCandidate(baseline, candidate, SPEC);
    expect(outcome.verdict).toBe(SKILL_REWRITE_VERDICT.hold);
    expect(outcome.candidateRate).toBeLessThan(outcome.baselineRate);
  });

  test("a short dev n is its own arm", () => {
    const baseline = scoreRewriteTranscripts(devTasks.slice(0, 1).map(passingSubmission), TASKS, {
      arm: "dev",
      spec: SPEC,
    });
    const candidate = scoreRewriteTranscripts(devTasks.slice(0, 1).map(passingSubmission), TASKS, {
      arm: "dev",
      spec: SPEC,
    });
    const outcome = selectRewriteCandidate(baseline, candidate, SPEC);
    expect(outcome.verdict).toBe(SKILL_REWRITE_VERDICT.not_enough_evidence);
  });

  test("a candidate promoting on a self-selected dev subset is not_enough_evidence", () => {
    // The A-1 exploit: baseline runs the whole receipted dev arm, the
    // candidate runs only its two easiest tasks. n = 2 meets the floor
    // and the subset rate is 1.0, but five receipted dev tasks were
    // never attempted, so the preregistered comparison cannot be
    // computed and the gate must not promote.
    const baseline = scoreRewriteTranscripts(devTasks.map(failingSubmission), TASKS, {
      arm: "dev",
      spec: SPEC,
    });
    const candidate = scoreRewriteTranscripts(devTasks.slice(0, 2).map(passingSubmission), TASKS, {
      arm: "dev",
      spec: SPEC,
    });
    const outcome = selectRewriteCandidate(baseline, candidate, SPEC);
    expect(outcome.verdict).toBe(SKILL_REWRITE_VERDICT.not_enough_evidence);
  });

  test("a dev arm missing one receipted member is not_enough_evidence even with n met", () => {
    const baseline = scoreRewriteTranscripts(devTasks.map(failingSubmission), TASKS, {
      arm: "dev",
      spec: SPEC,
    });
    const candidate = scoreRewriteTranscripts(devTasks.slice(1).map(passingSubmission), TASKS, {
      arm: "dev",
      spec: SPEC,
    });
    const outcome = selectRewriteCandidate(baseline, candidate, SPEC);
    expect(outcome.verdict).toBe(SKILL_REWRITE_VERDICT.not_enough_evidence);
  });

  test("a score outside the receipt's dev membership refuses the selection by name", () => {
    // Reachable only by bypassing the arm-checked scorer; the selection
    // refuses it the way the held-out gate refuses a non-member, rather
    // than letting an unregistered id move the metric.
    const heldoutTask = TASKS.tasks.find((task) => SPEC.heldoutTaskIds.includes(task.id))!;
    const baseline = scoreRewriteTranscripts(devTasks.map(passingSubmission), TASKS, {
      arm: "dev",
      spec: SPEC,
    });
    const strayCandidate = scoreRewriteTranscripts(
      [...devTasks.map(passingSubmission), passingSubmission(heldoutTask)],
      TASKS,
    );
    expect(() => selectRewriteCandidate(baseline, strayCandidate, SPEC)).toThrow(
      /not a dev member/,
    );
    const strayBaseline = scoreRewriteTranscripts([passingSubmission(heldoutTask)], TASKS);
    expect(() => selectRewriteCandidate(strayBaseline, baseline, SPEC)).toThrow(/not a dev member/);
  });
});

// Rates on this synthetic 8-id dev arm are exact binary fractions, so
// the boundary case is bit-exact by construction: baseline 2/8 = 0.25,
// candidate 4/8 = 0.5, margin 0.25, and 0.25 + 0.25 is exactly 0.5 in
// IEEE doubles. A `>` relaxed to `>=` promotes here; nothing else in
// the suite lands on the boundary.
const GATE_IDS = [
  "gate-01",
  "gate-02",
  "gate-03",
  "gate-04",
  "gate-05",
  "gate-06",
  "gate-07",
  "gate-08",
];

const gateScore = (id: string, passed: boolean): RewriteTaskScore => ({
  taskId: id,
  passed,
  failedChecks: [],
});

const gateArm = (passCount: number): ReadonlyArray<RewriteTaskScore> =>
  GATE_IDS.map((id, index) => gateScore(id, index < passCount));

const gateSpec = (margin: number): SkillRewriteSpec => ({
  ...SPEC,
  devTaskIds: GATE_IDS,
  requiredMargin: margin,
});

describe("selectRewriteCandidate strict-beat boundary", () => {
  test("a beat exactly at the required margin holds", () => {
    const outcome = selectRewriteCandidate(gateArm(2), gateArm(4), gateSpec(0.25));
    expect(outcome.baselineRate).toBe(0.25);
    expect(outcome.candidateRate).toBe(0.5);
    expect(outcome.verdict).toBe(SKILL_REWRITE_VERDICT.hold);
  });

  test("a real beat under the margin holds", () => {
    const outcome = selectRewriteCandidate(gateArm(2), gateArm(3), gateSpec(0.25));
    expect(outcome.candidateRate).toBeGreaterThan(outcome.baselineRate);
    expect(outcome.verdict).toBe(SKILL_REWRITE_VERDICT.hold);
  });

  test("a beat beyond the margin promotes", () => {
    const outcome = selectRewriteCandidate(gateArm(2), gateArm(5), gateSpec(0.25));
    expect(outcome.verdict).toBe(SKILL_REWRITE_VERDICT.promote);
  });
});

describe("checkRewriteHeldout", () => {
  const heldoutTasks = TASKS.tasks.filter((task) => SPEC.heldoutTaskIds.includes(task.id));

  test("a full held-out pass promotes", () => {
    const scores = scoreRewriteTranscripts(heldoutTasks.map(passingSubmission), TASKS, {
      arm: "heldout",
      spec: SPEC,
    });
    expect(checkRewriteHeldout(scores, SPEC)).toBe(SKILL_REWRITE_VERDICT.promote);
  });

  test("a held-out miss downgrades to hold", () => {
    const scores = scoreRewriteTranscripts(
      heldoutTasks.map((task, index) =>
        index === 0 ? failingSubmission(task) : passingSubmission(task),
      ),
      TASKS,
      { arm: "heldout", spec: SPEC },
    );
    expect(checkRewriteHeldout(scores, SPEC)).toBe(SKILL_REWRITE_VERDICT.hold);
  });

  test("a partial held-out pass refuses: every held-out member must be scored", () => {
    // A-2: one passing transcript for the first two members; t-05 was
    // never run. The one-shot check is incomplete, and an incomplete
    // check refuses by name (naming the unscored member) instead of
    // recording a hold the check never earned.
    const scores = scoreRewriteTranscripts(heldoutTasks.slice(0, 2).map(passingSubmission), TASKS, {
      arm: "heldout",
      spec: SPEC,
    });
    expect(() => checkRewriteHeldout(scores, SPEC)).toThrow(/t-05/);
    expect(() => checkRewriteHeldout(scores, SPEC)).toThrow(/incomplete/);
  });

  test("a dev score array handed to the held-out gate refuses by name", () => {
    const devTask = TASKS.tasks.find((task) => SPEC.devTaskIds.includes(task.id))!;
    const devScores = scoreRewriteTranscripts([passingSubmission(devTask)], TASKS, {
      arm: "dev",
      spec: SPEC,
    });
    expect(() => checkRewriteHeldout(devScores, SPEC)).toThrow(/not a held-out member/);
  });

  test("no held-out evidence refuses the check rather than passing or holding quietly", () => {
    // An empty arm is the degenerate partial coverage: zero of the
    // receipted members were scored, so the one-shot check refuses
    // instead of issuing any verdict.
    expect(() => checkRewriteHeldout([], SPEC)).toThrow(/incomplete/);
  });
});

describe("resolveSkillRewriteMode", () => {
  test("gated and iterative are accepted, absent resolves to gated", () => {
    expect(resolveSkillRewriteMode(undefined)).toBe("gated");
    expect(resolveSkillRewriteMode("")).toBe("gated");
    expect(resolveSkillRewriteMode("  ")).toBe("gated");
    expect(resolveSkillRewriteMode("gated")).toBe("gated");
    expect(resolveSkillRewriteMode("iterative")).toBe("iterative");
  });

  test("an unknown value is a named error", () => {
    expect(() => resolveSkillRewriteMode("turbo")).toThrow(/turbo/);
  });
});

describe("runRewriteJudge", () => {
  test("skipped without a command", () => {
    expect(runRewriteJudge(undefined, { metric: SKILL_REWRITE_METRIC }).status).toBe("skipped");
  });

  test("a failing command is an error status, never a throw", () => {
    const outcome = runRewriteJudge("exit 3", { metric: SKILL_REWRITE_METRIC });
    expect(outcome.status).toBe("error");
  });

  test("advisory verdicts never move the metric", () => {
    // The judge's verdict field is reported, never folded into selection:
    // selectRewriteCandidate takes only RewriteTaskScore arrays and the
    // spec, so no judge output type can reach it. Pinned by a run whose
    // judge output contradicts the deterministic scores.
    const devTasks = TASKS.tasks.filter((task) => SPEC.devTaskIds.includes(task.id));
    const baseline = scoreRewriteTranscripts(devTasks.map(failingSubmission), TASKS, {
      arm: "dev",
      spec: SPEC,
    });
    const candidate = scoreRewriteTranscripts(devTasks.map(passingSubmission), TASKS, {
      arm: "dev",
      spec: SPEC,
    });
    expect(selectRewriteCandidate(baseline, candidate, SPEC).verdict).toBe(
      SKILL_REWRITE_VERDICT.promote,
    );
  });
});

describe("buildRewriteGradeRecord", () => {
  test("carries ids, scores and check verdicts with the spec hash", () => {
    const devTasks = TASKS.tasks.filter((task) => SPEC.devTaskIds.includes(task.id));
    const heldoutTasks = TASKS.tasks.filter((task) => SPEC.heldoutTaskIds.includes(task.id));
    const baseline = scoreRewriteTranscripts(devTasks.map(failingSubmission), TASKS, {
      arm: "dev",
      spec: SPEC,
    });
    const candidate = scoreRewriteTranscripts(devTasks.map(passingSubmission), TASKS, {
      arm: "dev",
      spec: SPEC,
    });
    const selection = selectRewriteCandidate(baseline, candidate, SPEC);
    const heldout = scoreRewriteTranscripts(heldoutTasks.map(passingSubmission), TASKS, {
      arm: "heldout",
      spec: SPEC,
    });
    const verdict = checkRewriteHeldout(heldout, SPEC);
    // The candidate binding (review round-2, M1 residual R1): the final
    // promote rides a staging, and the record carries the sha-256 the
    // staged proposal stamps, so a promote grade can only bless the exact
    // document it was earned against.
    const candidateSha256 = sha256Hex(CANDIDATE_BODY);
    const record = buildRewriteGradeRecord({
      skill: TASKS.skillName,
      spec: SPEC,
      specHash: "spec-hash-value",
      selection,
      heldoutVerdict: verdict,
      devScores: candidate,
      heldoutScores: heldout,
      candidateSha256,
    });
    expect(record.skill).toBe(TASKS.skillName);
    expect(record.spec_hash).toBe("spec-hash-value");
    expect(record.metric).toBe(SKILL_REWRITE_METRIC);
    expect(record.verdict).toBe(SKILL_REWRITE_VERDICT.promote);
    expect(record.tasks_hash).toBe(SPEC.tasksHash);
    expect(record.candidate_sha256).toBe(candidateSha256);
    expect(record.dev.map((row) => row.task_id).toSorted()).toEqual(
      [...SPEC.devTaskIds].toSorted(),
    );
    expect(record.held_out.map((row) => row.task_id).toSorted()).toEqual(
      [...SPEC.heldoutTaskIds].toSorted(),
    );
    for (const row of record.dev) {
      expect(typeof row.passed).toBe("boolean");
      expect(Array.isArray(row.failed_checks)).toBe(true);
    }
  });

  test("the serialized record never carries task or transcript text", () => {
    const devTasks = TASKS.tasks.filter((task) => SPEC.devTaskIds.includes(task.id));
    const candidate = scoreRewriteTranscripts(devTasks.map(passingSubmission), TASKS, {
      arm: "dev",
      spec: SPEC,
    });
    const record = buildRewriteGradeRecord({
      skill: TASKS.skillName,
      spec: SPEC,
      specHash: "spec-hash-value",
      selection: selectRewriteCandidate(candidate, candidate, SPEC),
      heldoutVerdict: SKILL_REWRITE_VERDICT.hold,
      devScores: candidate,
      heldoutScores: [],
      candidateSha256: sha256Hex(CANDIDATE_BODY),
    });
    const serialized = JSON.stringify(record);
    for (const task of TASKS.tasks) {
      // Task input text, down to a distinguishing fragment, never ships.
      const fragment = task.input.slice(0, 24);
      expect(serialized).not.toContain(fragment);
    }
    // Transcript step and tool names are never echoed either: only the
    // check ids, the boolean verdicts and the rates travel.
    expect(serialized).not.toContain("note_write");
    expect(serialized).not.toContain("bulk_delete");
    // The candidate binding is a sha-256, not content: adding the hash to
    // the payload must not smuggle the candidate body in beside it. A
    // hold round (this one) stages nothing, but the field rides the same
    // assembly, so the no-text invariant is pinned over the extended
    // payload shape.
    expect(record.candidate_sha256).toBe(sha256Hex(CANDIDATE_BODY));
    expect(record.candidate_sha256).toMatch(/^[0-9a-f]{64}$/);
    expect(serialized).not.toContain(CANDIDATE_BODY.slice(0, 24));
  });
});