Local-first 🧠memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.
TypeScript
/**
* The rewrite gate kernels (agent-presence, Lane A).
*
* One gated rewrite round, kernel by kernel. OSB never calls a model:
* the envelope builders hand the calling agent one `needs-llm-step` per
* task - dev members only, until the single held-out check after a
* promote - and the calling agent executes and submits transcripts. The
* scorer validates the submitted shape and grades STRUCTURAL predicates
* only; it refuses a submission that carries a self-reported success
* field, so nothing the agent claims about its own work can move the
* metric. Selection is the preregistered three arms from the
* decision-model eval, and the held-out gate runs once, on the candidate
* that earned promotion; a miss downgrades it.
*
* Coverage is part of the preregistration. The dev arms may only decide
* when every dev member has a scored transcript on BOTH arms, and the
* held-out gate runs only on the full held-out membership. A short dev
* arm - an n under the floor, or receipted ids that were never
* submitted - is `not_enough_evidence`, the same honest outcome either
* way: the comparison the receipt preregistered cannot be computed yet,
* and the round stays re-runnable while the missing transcripts are
* produced. A score for an id OUTSIDE the receipt's membership is
* different in kind - the arm scored something the receipt never
* registered - and is refused by name. The held-out gate refuses
* instead of holding on partial coverage, because a hold would be
* recorded in a grade record as a verdict the one-shot check never
* earned (and would leak which held-out tasks fail without the check
* having run in full); a refusal writes nothing.
*
* Round transcripts are bound to the candidate body they were produced
* against: every envelope carries `candidate_sha256`, the sha-256 of the
* body in its prompt, the prompt names the binding, and the scorer
* refuses - by name - a transcript whose `candidate_sha256` is missing
* or stale whenever the caller hands it the current candidate body's
* hash. No transcript can gate a candidate it was never run against.
*
* Every refusal here is a named {@link SkillRewriteError}: a stale task
* set, a phantom receipt id, a malformed submission, a self-report, a
* cross-arm leak, a stale or missing candidate binding, an unregistered
* task id, an incomplete held-out check, an unknown mode - none of them
* is ever a silent skip, because a gate that quietly passes is worse
* than no gate.
*/
import { sha256Hex } from "../integrity/digest.ts";
import { runJsonCommandBridge } from "../reliability/command-bridge.ts";
import { buildNeedsLlmStep, type NeedsLlmStep } from "./llm-step.ts";
import {
SKILL_BENCH_CHECK,
skillBenchmarkRelPath,
SKILL_BENCHMARK_TASKS_FILE,
type SkillBenchExpectation,
type SkillBenchmarkTask,
type SkillBenchmarkTaskSet,
type SkillBenchValueClass,
} from "./skill-rewrite-tasks.ts";
import {
SKILL_REWRITE_METRIC,
SKILL_REWRITE_VERDICT,
type SkillRewriteSpec,
type SkillRewriteVerdict,
} from "./skill-rewrite-spec.ts";
// The verdict vocabulary rides the gate outputs: the CLI and MCP faces
// read it from here, beside the verdicts they print.
export {
isSkillRewriteVerdict,
SKILL_REWRITE_VERDICT,
SKILL_REWRITE_VERDICTS,
} from "./skill-rewrite-spec.ts";
/** The step name a rewrite task envelope answers. */
export const SKILL_REWRITE_STEP = "skill-rewrite-task";
/** Subdirectory of the skill's benchmark folder holding round transcripts. */
export const SKILL_REWRITE_ROUNDS_DIR = "rounds";
/** Which round an envelope belongs to. The dev rounds are the baseline
* (incumbent body) and the candidate; `heldout` exists but is never a
* dev round - its envelope builder is the single post-promotion check. */
export const SKILL_REWRITE_ROUND_ARM = Object.freeze({
baseline: "baseline",
candidate: "candidate",
heldout: "heldout",
} as const);
export type SkillRewriteRoundArm =
(typeof SKILL_REWRITE_ROUND_ARM)[keyof typeof SKILL_REWRITE_ROUND_ARM];
/** The round-arm members, for parse-time refusal and guard sharing. */
export const SKILL_REWRITE_ROUND_ARMS: ReadonlyArray<SkillRewriteRoundArm> = Object.freeze(
Object.values(SKILL_REWRITE_ROUND_ARM),
);
/**
* One rewrite round envelope: the needs-llm-step spine plus the fields
* that tie the generated transcript back to the task it answers, the arm
* that asked for it, and the candidate body the task was rendered
* against. The submission scorer reads `task_id` and `candidate_sha256`
* off the TRANSCRIPT, not the envelope; these fields exist so a caller
* can correlate an envelope with its task, so the executing agent knows
* exactly which hash the transcript must carry, and so the dev-only
* containment assertion in the builder has a typed handle.
*/
export type RewriteTaskEnvelope = NeedsLlmStep & {
readonly task_id: string;
readonly round: SkillRewriteRoundArm;
/** sha-256 of the candidate body in this envelope's prompt; the
* transcript must echo it, and the scorer refuses a stale one. */
readonly candidate_sha256: string;
};
export function isSkillRewriteRoundArm(value: unknown): value is SkillRewriteRoundArm {
return (
typeof value === "string" && (SKILL_REWRITE_ROUND_ARMS as ReadonlyArray<string>).includes(value)
);
}
/** The closed rewrite-mode vocabulary. `gated` is one candidate per
* round; `iterative` reserves the explicitly opted-in bounded edit loop
* and is accepted but not implemented yet, so it currently resolves to
* the same single-candidate behavior. Both are gated before a write. */
export const SKILL_REWRITE_MODE = Object.freeze({
gated: "gated",
iterative: "iterative",
} as const);
export type SkillRewriteMode = (typeof SKILL_REWRITE_MODE)[keyof typeof SKILL_REWRITE_MODE];
/** The mode members, for guard sharing. */
export const SKILL_REWRITE_MODES: ReadonlyArray<SkillRewriteMode> = Object.freeze(
Object.values(SKILL_REWRITE_MODE),
);
export function isSkillRewriteMode(value: unknown): value is SkillRewriteMode {
return (
typeof value === "string" && (SKILL_REWRITE_MODES as ReadonlyArray<string>).includes(value)
);
}
/** A gate refusal. Every refusal in this module is this error, by name. */
export class SkillRewriteError extends Error {
constructor(message: string) {
super(message);
this.name = "SkillRewriteError";
}
}
/** One scored transcript: which dev or held-out task, and which checks failed. */
export interface RewriteTaskScore {
readonly taskId: string;
readonly passed: boolean;
/** Check ids the transcript failed, deduped, in expectation order. */
readonly failedChecks: ReadonlyArray<string>;
}
/**
* One submitted transcript, in its serialized shape - the exact JSON the
* envelope's schema hints name, dropped at the envelope's target path.
* Field spellings are snake case to match the file on disk.
* `candidate_sha256` binds the transcript to the candidate body it was
* produced against; when the caller hands the scorer the current
* candidate hash, a transcript without it - or with a stale one - is
* refused, so scores from a previous body never gate this one.
*/
export interface RewriteSubmission {
readonly task_id: string;
readonly steps?: ReadonlyArray<string>;
readonly tools_used?: ReadonlyArray<string>;
readonly artifacts?: ReadonlyArray<string>;
readonly output?: Readonly<Record<string, unknown>>;
readonly candidate_sha256?: string;
}
/** The selection gate's outcome, with the rates it decided on. */
export interface RewriteSelection {
readonly verdict: SkillRewriteVerdict;
readonly baselineRate: number;
readonly candidateRate: number;
}
/**
* Top-level submission keys that report the agent's own success. They are
* refused, not ignored: ignoring them would let a submitter believe the
* field meant something, and accepting them would put a self-report one
* refactor away from the metric.
*/
const SELF_REPORTED_KEYS: ReadonlyArray<string> = Object.freeze([
"success",
"passed",
"succeeded",
"ok",
]);
/** The closed top-level key set a submission may carry. */
const KNOWN_SUBMISSION_KEYS: ReadonlyArray<string> = Object.freeze([
"task_id",
"steps",
"tools_used",
"artifacts",
"output",
"candidate_sha256",
]);
/** A transcript's `candidate_sha256` must be a sha-256 hex digest. */
const SHA256_HEX_RE = /^[0-9a-f]{64}$/;
/** Dev rounds an envelope builder may be asked for. */
type DevRound = Extract<SkillRewriteRoundArm, "baseline" | "candidate">;
/**
* Build one envelope per DEV task. The candidate body and the task input
* travel in the prompt; the target path names the transcript drop site
* under the skill's benchmark rounds tree. Held-out members are refused
* by construction: the builder walks `spec.devTaskIds` and nothing else,
* and a task set whose hash no longer matches the receipt is refused
* before any envelope exists.
*/
export function buildRewriteEnvelopes(
candidateBody: string,
spec: SkillRewriteSpec,
tasks: SkillBenchmarkTaskSet,
opts: { readonly round?: DevRound } = {},
): ReadonlyArray<RewriteTaskEnvelope> {
const round = opts.round ?? SKILL_REWRITE_ROUND_ARM.candidate;
return buildRoundEnvelopes(candidateBody, spec, tasks, spec.devTaskIds, round);
}
/**
* The ONE held-out envelope builder, and the only place held-out task
* text may enter a prompt: after the dev selection promoted a candidate,
* the held-out check runs once, and its results are never fed back.
*/
export function buildRewriteHeldoutEnvelopes(
candidateBody: string,
spec: SkillRewriteSpec,
tasks: SkillBenchmarkTaskSet,
): ReadonlyArray<RewriteTaskEnvelope> {
return buildRoundEnvelopes(
candidateBody,
spec,
tasks,
spec.heldoutTaskIds,
SKILL_REWRITE_ROUND_ARM.heldout,
);
}
function buildRoundEnvelopes(
candidateBody: string,
spec: SkillRewriteSpec,
tasks: SkillBenchmarkTaskSet,
memberIds: ReadonlyArray<string>,
round: SkillRewriteRoundArm,
): ReadonlyArray<RewriteTaskEnvelope> {
if (tasks.hash !== spec.tasksHash) {
throw new SkillRewriteError(
`skill rewrite: task set hash ${tasks.hash} does not match the receipt's frozen hash ` +
`${spec.tasksHash}; the benchmark changed under the spec`,
);
}
const byId = new Map(tasks.tasks.map((task) => [task.id, task]));
const devMembers = new Set(spec.devTaskIds);
const candidateHash = sha256Hex(candidateBody);
const envelopes = memberIds.map((taskId) => {
const task = byId.get(taskId);
if (task === undefined) {
throw new SkillRewriteError(
`skill rewrite: receipt names task id '${taskId}' but the task set for ` +
`'${spec.skillName}' has no such task (${SKILL_BENCHMARK_TASKS_FILE} is stale or hand-edited)`,
);
}
const targetPath = skillBenchmarkRelPath(
spec.skillName,
SKILL_REWRITE_ROUNDS_DIR,
round,
`${task.id}.json`,
);
return buildNeedsLlmStep({
step: SKILL_REWRITE_STEP,
prompt: renderTaskPrompt(candidateBody, task, targetPath, candidateHash),
schema_hints: Object.freeze([SUBMISSION_SHAPE_HINT]),
target_path: targetPath,
task_id: task.id,
round,
candidate_sha256: candidateHash,
});
});
// The containment assertion the gate leans on: every envelope answers a
// dev member. True by construction here, asserted so a future edit that
// walks the wrong list fails here instead of leaking held-out text.
const leaking = envelopes.filter((envelope) => !devMembers.has(String(envelope["task_id"])));
if (leaking.length > 0 && round !== SKILL_REWRITE_ROUND_ARM.heldout) {
throw new SkillRewriteError(
`skill rewrite: envelope(s) for non-dev task(s) ${leaking
.map((envelope) => String(envelope["task_id"]))
.join(", ")} refused; held-out members never enter a dev round`,
);
}
return envelopes;
}
/** The transcript contract the submitting agent must honor. */
const SUBMISSION_SHAPE_HINT =
"Return one JSON object per task with keys: task_id (string), steps (string array), " +
"tools_used (string array), artifacts (string array), output (object), " +
"candidate_sha256 (string, the sha-256 hex digest of the candidate skill body the " +
"task was performed against). " +
"No other top-level keys.";
function renderTaskPrompt(
candidateBody: string,
task: SkillBenchmarkTask,
targetPath: string,
candidateHash: string,
): string {
return [
"Perform the task exactly as the candidate skill body instructs, then record what you did.",
"",
"Candidate skill body:",
candidateBody,
"",
`Task ${task.id}:`,
task.input,
"",
`Set "candidate_sha256" to ${candidateHash} in the transcript: it binds the ` +
"transcript to the candidate body above, and a transcript for a different body " +
"is refused.",
"",
`Write the JSON transcript to ${targetPath} when the task is done.`,
].join("\n");
}
/** Options for {@link scoreRewriteTranscripts}: which arm is being scored,
* against which receipt, and which candidate body the transcripts must be
* bound to. Omitted opts scores whatever was submitted, unbound. */
export interface ScoreRewriteOptions {
readonly arm: "dev" | "heldout";
readonly spec: SkillRewriteSpec;
/** The sha-256 of the CURRENT candidate body. When given, every
* submission must carry a matching `candidate_sha256`; a missing or
* stale binding is a named refusal, so transcripts left over from a
* previous candidate can never gate this one. */
readonly candidateHash?: string;
}
/**
* Validate submitted transcripts and score them against the task set's
* structural predicates. Throws a named {@link SkillRewriteError} on any
* malformed shape, an unknown task id, a duplicate submission, a
* self-reported success field, a malformed or stale `candidate_sha256`
* binding, or - when the arm is named - a submission for a task the arm
* may not see. Scores come back sorted by task id.
*/
export function scoreRewriteTranscripts(
submissions: ReadonlyArray<unknown>,
tasks: SkillBenchmarkTaskSet,
opts?: ScoreRewriteOptions,
): ReadonlyArray<RewriteTaskScore> {
const byId = new Map(tasks.tasks.map((task) => [task.id, task]));
const devMembers = new Set(opts?.spec.devTaskIds ?? []);
const heldoutMembers = new Set(opts?.spec.heldoutTaskIds ?? []);
const seen = new Set<string>();
const scores: RewriteTaskScore[] = [];
for (const raw of submissions) {
if (raw === null || typeof raw !== "object" || Array.isArray(raw)) {
throw new SkillRewriteError("rewrite submission must be an object");
}
const record = raw as Record<string, unknown>;
const taskId = record["task_id"];
if (typeof taskId !== "string" || taskId.trim().length === 0) {
throw new SkillRewriteError("rewrite submission needs a task_id");
}
const task = byId.get(taskId);
if (task === undefined) {
throw new SkillRewriteError(`rewrite submission names an unknown task id: ${taskId}`);
}
for (const key of SELF_REPORTED_KEYS) {
if (key in record) {
throw new SkillRewriteError(
`rewrite submission carries a self-reported success field ('${key}'); ` +
"the scorer reads structural predicates only",
);
}
}
for (const key of Object.keys(record)) {
if (!(KNOWN_SUBMISSION_KEYS as ReadonlyArray<string>).includes(key)) {
throw new SkillRewriteError(
`rewrite submission carries an unexpected key ('${key}'); ` +
`the transcript shape is ${KNOWN_SUBMISSION_KEYS.join(", ")}`,
);
}
}
const boundHash = record["candidate_sha256"];
if (
boundHash !== undefined &&
(typeof boundHash !== "string" || !SHA256_HEX_RE.test(boundHash))
) {
throw new SkillRewriteError(
"rewrite submission candidate_sha256 must be a sha-256 hex digest of the " +
"candidate body the task was performed against",
);
}
if (opts?.candidateHash !== undefined) {
if (boundHash === undefined) {
throw new SkillRewriteError(
`rewrite submission for task '${taskId}' is missing candidate_sha256; transcripts ` +
`are bound to the candidate body they were run against (expected ${opts.candidateHash})`,
);
}
if (boundHash !== opts.candidateHash) {
throw new SkillRewriteError(
`rewrite submission for task '${taskId}' carries candidate_sha256 ${boundHash}, ` +
"which does not match the current candidate body " +
`(${opts.candidateHash}); a transcript from a different body never gates this one`,
);
}
}
if (seen.has(taskId)) {
throw new SkillRewriteError(`rewrite submission for task '${taskId}' appears twice`);
}
seen.add(taskId);
if (opts !== undefined) {
const memberOfArm = opts.arm === "dev" ? devMembers : heldoutMembers;
const otherArm = opts.arm === "dev" ? heldoutMembers : devMembers;
if (!memberOfArm.has(taskId)) {
const why = otherArm.has(taskId) ? "a held-out task" : "a task outside this arm";
const armName = opts.arm === "dev" ? "dev" : "held-out";
throw new SkillRewriteError(
`rewrite submission for '${taskId}' refused in the ${armName} arm: ` +
`'${taskId}' is ${why}` +
(otherArm.has(taskId) && opts.arm === "dev"
? "; held-out members are scored by the single held-out check only"
: "; dev members are scored in the dev arm only"),
);
}
}
const steps = stringList(record["steps"], "steps");
const toolsUsed = stringList(record["tools_used"], "tools_used");
const artifacts = stringList(record["artifacts"], "artifacts");
const outputRaw = record["output"] ?? {};
if (outputRaw === null || typeof outputRaw !== "object" || Array.isArray(outputRaw)) {
throw new SkillRewriteError("rewrite submission output must be an object");
}
const output = outputRaw as Readonly<Record<string, unknown>>;
scores.push(scoreTask(task, { steps, toolsUsed, artifacts, output }));
}
return scores.toSorted((left, right) => left.taskId.localeCompare(right.taskId));
}
/**
* The selection gate, preregistered three arms: a candidate promotes only
* by strictly beating the baseline by more than the spec's margin with
* the dev n met; a tie - and anything under the margin - holds; a short
* dev n is its own arm, never a pass.
*
* "Met" means the receipt's full dev membership, scored on BOTH arms: a
* score for every dev task id. An arm that scored a subset - however
* large, and however good the subset's rate - has not produced the
* preregistered comparison, so the verdict is `not_enough_evidence`,
* exactly as for an n under the floor: both are the same shortfall
* (incomplete evidence for the receipted claim), and folding them keeps
* a mid-round invocation a verdict instead of an error, re-runnable once
* the missing transcripts land. A score for an id OUTSIDE the receipt's
* dev membership is different in kind - the arm scored something the
* receipt never registered - and is refused by name, matching the
* held-out gate's member refusal.
*/
export function selectRewriteCandidate(
baseline: ReadonlyArray<RewriteTaskScore>,
candidate: ReadonlyArray<RewriteTaskScore>,
spec: SkillRewriteSpec,
): RewriteSelection {
assertDevMembers(baseline, "baseline", spec);
assertDevMembers(candidate, "candidate", spec);
const n = Math.min(baseline.length, candidate.length);
const baselineRate = passedRate(baseline);
const candidateRate = passedRate(candidate);
const fullyScored = coversAll(baseline, spec.devTaskIds) && coversAll(candidate, spec.devTaskIds);
if (!fullyScored || n < spec.minDevTasks) {
return {
verdict: SKILL_REWRITE_VERDICT.not_enough_evidence,
baselineRate,
candidateRate,
};
}
if (candidateRate > baselineRate + spec.requiredMargin) {
return { verdict: SKILL_REWRITE_VERDICT.promote, baselineRate, candidateRate };
}
return { verdict: SKILL_REWRITE_VERDICT.hold, baselineRate, candidateRate };
}
/**
* The dev arms score receipt members only: an id outside `spec.devTaskIds`
* in either arm means the caller bypassed the arm-checked scorer, so the
* selection refuses by name instead of letting an unregistered id move
* the metric.
*/
function assertDevMembers(
scores: ReadonlyArray<RewriteTaskScore>,
armName: string,
spec: SkillRewriteSpec,
): void {
const devMembers = new Set(spec.devTaskIds);
for (const score of scores) {
if (!devMembers.has(score.taskId)) {
throw new SkillRewriteError(
`rewrite selection scored '${score.taskId}' in the ${armName} arm, which is not a ` +
"dev member of the receipt; the dev arms score the receipt's dev members only",
);
}
}
}
/** Whether every receipted id has a scored transcript in the arm. */
function coversAll(
scores: ReadonlyArray<RewriteTaskScore>,
memberIds: ReadonlyArray<string>,
): boolean {
const scored = new Set(scores.map((score) => score.taskId));
return memberIds.every((id) => scored.has(id));
}
/**
* The held-out gate, run ONCE, on the candidate that earned promotion: a
* full pass promotes, any miss holds. Every score must name a held-out
* member (a dev score array handed here is refused by name rather than
* folded in), and every held-out member must have a score: the one-shot
* check is incomplete until the whole preregistered membership has run.
* An incomplete check refuses by name - naming the unscored members -
* rather than hold, because a hold would be recorded in a grade record
* as a verdict the check never earned, and would tell the submitting
* agent which held-out tasks fail without the check having run in full.
* An empty arm is the degenerate partial coverage (zero of the
* receipted members scored) and refuses the same way; no verdict at all
* is the only honest output of a check that did not run.
*/
export function checkRewriteHeldout(
candidate: ReadonlyArray<RewriteTaskScore>,
spec: SkillRewriteSpec,
): SkillRewriteVerdict {
const heldoutMembers = new Set(spec.heldoutTaskIds);
for (const score of candidate) {
if (!heldoutMembers.has(score.taskId)) {
throw new SkillRewriteError(
`held-out gate scored '${score.taskId}', which is not a held-out member of the receipt; ` +
"the held-out arm scores held-out members only",
);
}
}
const scored = new Set(candidate.map((score) => score.taskId));
const unscored = spec.heldoutTaskIds.filter((id) => !scored.has(id));
if (unscored.length > 0) {
throw new SkillRewriteError(
`the held-out check is incomplete: ${unscored.length} of ${spec.heldoutTaskIds.length} ` +
`held-out members were never scored (${unscored.join(", ")}); the one-shot check runs ` +
"on the full receipted membership or issues no verdict at all",
);
}
return candidate.every((score) => score.passed)
? SKILL_REWRITE_VERDICT.promote
: SKILL_REWRITE_VERDICT.hold;
}
/**
* Resolve the rewrite mode. Absent (or blank) resolves to `gated`, the
* documented default; `gated` and `iterative` pass through; anything
* else is a named error, never a silent fallback to the default.
*/
export function resolveSkillRewriteMode(raw: string | undefined): SkillRewriteMode {
const value = raw?.trim() ?? "";
if (value === "") return SKILL_REWRITE_MODE.gated;
if (isSkillRewriteMode(value)) return value;
throw new SkillRewriteError(
`skill rewrite mode '${value}' is unknown (expected one of ${SKILL_REWRITE_MODES.join(", ")})`,
);
}
// ----- the advisory judge -------------------------------------------------------
export interface RewriteJudgeOutcome {
readonly status: "skipped" | "error" | "ran";
readonly detail?: string;
/** Advisory per-task verdicts; never folded into the metric. */
readonly verdicts?: Readonly<Record<string, boolean>>;
}
/**
* Optional advisory judge over the score summary, through the one
* sanctioned external-command boundary. The deterministic scorer stays
* canonical in every message and every doc line: the judge's verdicts are
* reported for comparison and can never reach
* {@link selectRewriteCandidate}, whose signature takes score arrays and
* a spec only. A failing command is an error status, never a throw.
*/
export function runRewriteJudge(
cmd: string | undefined,
summary: Record<string, unknown>,
): RewriteJudgeOutcome {
const result = runJsonCommandBridge(cmd, summary, { label: "skill rewrite judge command" });
if (result.status === "skipped") return Object.freeze({ status: "skipped" });
if (result.status === "error") {
return Object.freeze({ status: "error", detail: result.detail });
}
if (result.output === null || typeof result.output !== "object") {
return Object.freeze({ status: "error", detail: "judge output is not an object" });
}
const parsed = result.output as { verdicts?: Record<string, unknown> };
const verdicts: Record<string, boolean> = {};
for (const [id, value] of Object.entries(parsed.verdicts ?? {})) {
if (typeof value === "boolean") verdicts[id] = value;
}
return Object.freeze({ status: "ran", verdicts: Object.freeze(verdicts) });
}
// ----- the grade record ---------------------------------------------------------
/** One per-task row of a grade record: ids and check verdicts only. */
export interface RewriteGradeTaskRow {
readonly task_id: string;
readonly passed: boolean;
readonly failed_checks: ReadonlyArray<string>;
}
/**
* The `skill_rewrite_eval_grade` payload: identifiers, numbers, booleans
* and check ids, stamped with the spec hash. NEVER task text and NEVER
* transcript text - the payload is assembled from score rows, which carry
* no text by construction, and tests pin that over the serialized form.
* The one digest it may carry (`candidate_sha256`) binds the record to
* the staged proposal's bytes; a sha-256 is an identifier, not content.
*/
export interface RewriteGradeRecordPayload {
readonly skill: string;
readonly spec_hash: string;
readonly tasks_hash: string;
readonly metric: typeof SKILL_REWRITE_METRIC;
readonly seed: number;
readonly split_rule: string;
/** The final verdict: the selection verdict after the held-out gate. */
readonly verdict: SkillRewriteVerdict;
readonly baseline_rate: number;
readonly candidate_rate: number;
readonly dev: ReadonlyArray<RewriteGradeTaskRow>;
readonly held_out: ReadonlyArray<RewriteGradeTaskRow>;
/**
* The candidate binding (review round-2, M1 residual R1): the sha-256
* of the staged proposal document the final promote blessed, present
* ONLY when this record rode a staging - a hold round stages nothing
* and binds nothing. The accept-time grade match requires it, so a
* historical promote over a different candidate cannot bless a new
* proposal: rates collide, document digests do not.
*/
readonly candidate_sha256?: string;
}
export interface RewriteGradeRecordInput {
readonly skill: string;
readonly spec: SkillRewriteSpec;
/** The spec hash stamped on the record and the staged proposal. */
readonly specHash: string;
readonly selection: RewriteSelection;
/** The held-out gate's verdict; `hold` when it has not run yet. */
readonly heldoutVerdict: SkillRewriteVerdict;
readonly devScores: ReadonlyArray<RewriteTaskScore>;
readonly heldoutScores: ReadonlyArray<RewriteTaskScore>;
/**
* The staged proposal's document sha-256 - exactly what staging stamped
* and returned; omit on a hold round, which staged nothing. The accept
* match refuses a promote record without the binding, so a final
* promote whose caller drops this field fails CLOSED at the accept,
* never open.
*/
readonly candidateSha256?: string;
}
/**
* Assemble the grade-record payload for a completed gate run. The final
* verdict applies the one rule the held-out gate owns: a promoted
* candidate that misses held-out is downgraded to hold. A hold and a
* short dev n are already final from selection.
*/
export function buildRewriteGradeRecord(input: RewriteGradeRecordInput): RewriteGradeRecordPayload {
const verdict =
input.selection.verdict === SKILL_REWRITE_VERDICT.promote
? input.heldoutVerdict
: input.selection.verdict;
return Object.freeze({
skill: input.skill,
spec_hash: input.specHash,
tasks_hash: input.spec.tasksHash,
metric: SKILL_REWRITE_METRIC,
seed: input.spec.seed,
split_rule: input.spec.splitRule,
verdict,
baseline_rate: input.selection.baselineRate,
candidate_rate: input.selection.candidateRate,
dev: input.devScores.map(toGradeRow),
held_out: input.heldoutScores.map(toGradeRow),
...(input.candidateSha256 !== undefined ? { candidate_sha256: input.candidateSha256 } : {}),
});
}
function toGradeRow(score: RewriteTaskScore): RewriteGradeTaskRow {
return Object.freeze({
task_id: score.taskId,
passed: score.passed,
failed_checks: Object.freeze([...score.failedChecks]),
});
}
// ----- the structural predicates -------------------------------------------------
interface NormalizedTranscript {
readonly steps: ReadonlyArray<string>;
readonly toolsUsed: ReadonlyArray<string>;
readonly artifacts: ReadonlyArray<string>;
readonly output: Readonly<Record<string, unknown>>;
}
function scoreTask(task: SkillBenchmarkTask, transcript: NormalizedTranscript): RewriteTaskScore {
const failed: string[] = [];
for (const expectation of task.expects) {
const passed = evaluateExpectation(expectation, transcript);
if (!passed && !failed.includes(expectation.check)) {
failed.push(expectation.check);
}
}
return Object.freeze({
taskId: task.id,
passed: failed.length === 0,
failedChecks: Object.freeze(failed),
});
}
function evaluateExpectation(
expectation: SkillBenchExpectation,
transcript: NormalizedTranscript,
): boolean {
switch (expectation.check) {
case SKILL_BENCH_CHECK.tool_used:
return transcript.toolsUsed.includes(expectation.tool);
case SKILL_BENCH_CHECK.artifact_created: {
const prefix = normalizePath(expectation.path_prefix);
return transcript.artifacts.some((artifact) => normalizePath(artifact).startsWith(prefix));
}
case SKILL_BENCH_CHECK.step_order:
return isSubsequence(expectation.steps, transcript.steps);
case SKILL_BENCH_CHECK.forbidden_step:
return !transcript.steps.includes(expectation.step);
case SKILL_BENCH_CHECK.output_field:
return outputValueMatches(transcript.output[expectation.field], expectation.value_class);
}
}
function isSubsequence(needles: ReadonlyArray<string>, haystack: ReadonlyArray<string>): boolean {
let cursor = 0;
for (const step of haystack) {
if (cursor < needles.length && step === needles[cursor]) cursor++;
}
return cursor === needles.length;
}
function outputValueMatches(value: unknown, valueClass: SkillBenchValueClass): boolean {
switch (valueClass) {
case "non_empty_string":
return typeof value === "string" && value.trim().length > 0;
case "number":
return typeof value === "number" && Number.isFinite(value);
case "boolean":
return typeof value === "boolean";
case "non_empty_array":
return Array.isArray(value) && value.length > 0;
}
}
/** Absent means empty; present means a string array, else a named error. */
function stringList(raw: unknown, field: string): ReadonlyArray<string> {
if (raw === undefined) return Object.freeze([]);
if (!Array.isArray(raw) || !raw.every((entry) => typeof entry === "string")) {
throw new SkillRewriteError(`rewrite submission ${field} must be a string array`);
}
return Object.freeze(raw as string[]);
}
/** Artifact path shapes compare on forward slashes on every platform. */
function normalizePath(path: string): string {
return path.replaceAll("\\", "/");
}
function passedRate(scores: ReadonlyArray<RewriteTaskScore>): number {
if (scores.length === 0) return 0;
return scores.filter((score) => score.passed).length / scores.length;
}