Skip to content

A self-organizing Obsidian vault that gives AI coding agents persistent memory.

fix: default the delivery gate to a full model id, and say which model answered (#396)

Brenno Ferrarifb987888d37d
diff --git a/.claude/scripts/tests/delivery-gate.test.ts b/.claude/scripts/tests/delivery-gate.test.ts
index 74aaa6a..531c77f 100644
--- a/.claude/scripts/tests/delivery-gate.test.ts
+++ b/.claude/scripts/tests/delivery-gate.test.ts
@@ -30,6 +30,10 @@ import {
quotes,
selfTestOutcome,
sessionDirs,
+ DEFAULT_MODEL,
+ modelsRan,
+ modelNote,
+ ranAsAsked,
subagentReport,
SUBAGENT_TASK,
type Step,
@@ -524,3 +528,42 @@ describe("delivery gate: self-test and environment", () => {
assert.equal(both.length, 4, "the created and the resolved path are both slugged");
});
});
+
+/**
+ * The model the gate asks for, and the one that answered (#346). A bare alias
+ * is not honoured by the CLI for every model: `--model haiku` silently ran
+ * another model (recorded on the `reason` tool), so a gate labelled "opus"
+ * could judge delivery on something else. The default is a full model id, and
+ * the gate reads the model each session reports so a mismatch shows.
+ */
+describe("the gate's model", () => {
+ test("the default is a full model id, not a bare alias", () => {
+ assert.match(DEFAULT_MODEL, /^claude-[a-z]+-\d/);
+ assert.ok(!["opus", "sonnet", "haiku"].includes(DEFAULT_MODEL));
+ });
+
+ test("modelsRan reads the model each session's init event reports, once each", () => {
+ const stream = [
+ JSON.stringify({ type: "system", subtype: "init", model: "claude-opus-5-5" }),
+ JSON.stringify({ type: "assistant", message: { content: [] } }),
+ JSON.stringify({ type: "system", subtype: "init", model: "claude-opus-5-5" }),
+ "not json",
+ JSON.stringify({ type: "system", subtype: "hook_response", model: "ignored" }),
+ ].join("\n");
+ assert.deepEqual(modelsRan(stream), ["claude-opus-5-5"]);
+ assert.deepEqual(modelsRan(""), []);
+ });
+
+ test("modelNote names the model that answered, and flags one that isn't the model asked for", () => {
+ assert.equal(modelNote("claude-opus-5-5", ["claude-opus-5-5"]), "Model that answered: claude-opus-5-5.");
+ assert.match(modelNote("opus", ["claude-sonnet-5"]), /Asked for opus, but the sessions ran claude-sonnet-5: these verdicts are for claude-sonnet-5/);
+ assert.match(modelNote("claude-opus-5-5", []), /Could not tell which model answered/);
+ });
+
+ test("ranAsAsked: a run that can't show it ran the model asked for can't be judged", () => {
+ assert.equal(ranAsAsked("claude-opus-5-5", ["claude-opus-5-5"]), true);
+ assert.equal(ranAsAsked("opus", ["claude-sonnet-5"]), false);
+ assert.equal(ranAsAsked("claude-opus-5-5", ["claude-opus-5-5", "claude-sonnet-5"]), false);
+ assert.equal(ranAsAsked("claude-opus-5-5", []), false, "no model reported");
+ });
+});
diff --git a/.github/scripts/delivery-gate.ts b/.github/scripts/delivery-gate.ts
index 26bcafa..bd5416b 100644
--- a/.github/scripts/delivery-gate.ts
+++ b/.github/scripts/delivery-gate.ts
@@ -39,7 +39,7 @@
* Run it by hand on any Claude Code version before it is recommended, with a
* logged-in `claude`:
*
- * node --experimental-strip-types .github/scripts/delivery-gate.ts [--trusted-dir <dir>] [--file-issue] [--hooks-only] [--model opus] [--claude <bin>]
+ * node --experimental-strip-types .github/scripts/delivery-gate.ts [--trusted-dir <dir>] [--file-issue] [--hooks-only] [--model <full model id>] [--claude <bin>]
*
* `--trusted-dir` loads the mod the way a user gets it: found in the vault's
* `.claude/skills/` after the workspace-trust prompt, which a headless session
@@ -89,6 +89,12 @@ const FIXTURE_MIN_CHARS = Math.round(HOOK_CAP_CHARS * 1.2);
const SESSION_BUDGET_USD = "5";
/** How long one turn may take before the session is abandoned. */
const TURN_TIMEOUT_MS = 5 * 60_000;
+/**
+ * The model the gate asks for when `--model` isn't given: a full id, because a
+ * bare alias is not honoured for every model and can silently run another
+ * (#346). Update it when a newer model ships.
+ */
+export const DEFAULT_MODEL = "claude-opus-5-5";
/** What a checkpoint's answer must contain: the context's last line, and the middle marker when the context carries one. */
export type Expected = { readonly line: string; readonly marker: string | null };
@@ -278,6 +284,7 @@ type StreamEvent = {
tool_use_result?: { content?: unknown; totalToolUseCount?: number; agentType?: string };
is_error?: boolean;
result?: string;
+ model?: string;
};
const textOf = (content: unknown): string =>
@@ -296,6 +303,42 @@ export function subagentReport(handedBack: string): string {
return (end < 0 ? body : body.slice(0, end)).trim();
}
+/**
+ * The models a session's stream says ran: the `model` of each `system`/`init`
+ * event, once each, in order. Read so the gate can say what actually answered:
+ * a bare alias is not honoured for every model, and a mismatch would judge
+ * delivery on one model while the run is labelled with another (#346).
+ */
+export function modelsRan(streamJson: string): string[] {
+ const seen: string[] = [];
+ for (const line of streamJson.split("\n")) {
+ let event: StreamEvent;
+ try {
+ event = JSON.parse(line);
+ } catch {
+ continue;
+ }
+ if (event.type === "system" && event.subtype === "init" && typeof event.model === "string" && !seen.includes(event.model)) {
+ seen.push(event.model);
+ }
+ }
+ return seen;
+}
+
+/** Whether every session reported running the model asked for. */
+export function ranAsAsked(asked: string, ran: readonly string[]): boolean {
+ return ran.length > 0 && ran.every((m) => m === asked);
+}
+
+/** The line that says which model answered, and flags one that isn't the model asked for. */
+export function modelNote(asked: string, ran: readonly string[]): string {
+ if (ran.length === 0) return `Could not tell which model answered (asked for ${asked}).`;
+ const other = ran.filter((m) => m !== asked);
+ return other.length === 0
+ ? `Model that answered: ${asked}.`
+ : `⚠️ Asked for ${asked}, but the sessions ran ${ran.join(", ")}: these verdicts are for ${other.join(", ")}.`;
+}
+
/** Split a stream-json transcript into turns, one per `result` event. Events before the first result belong to the first turn. */
export function parseTurns(streamJson: string): Turn[] {
const turns: Turn[] = [];
@@ -759,7 +802,11 @@ function session(
}
/** One fresh fixture per session, checked before any turn is spent on it. */
-async function runSession(steps: readonly Step[], breakage: Breakage, options: SessionOptions): Promise<Verdict[]> {
+async function runSession(
+ steps: readonly Step[],
+ breakage: Breakage,
+ options: SessionOptions,
+): Promise<{ verdicts: Verdict[]; models: string[] }> {
const marker = newMarker();
const vault = buildFixture(breakage, { marker, trustedDir: options.trustedDir, withMod: options.withMod });
try {
@@ -784,12 +831,13 @@ async function runSession(steps: readonly Step[], breakage: Breakage, options: S
expectedByStep[index] = expectFor(step);
});
// Steps never sent (the session stopped early) are INVALID whatever their expectation.
- return judge(
+ const verdicts = judge(
steps,
parseTurns(transcript),
steps.map((_, i) => expectedByStep[i] ?? ""),
{ withMod: options.withMod, ending },
);
+ return { verdicts, models: modelsRan(transcript) };
} finally {
removeFixture(vault);
}
@@ -861,12 +909,18 @@ async function main(): Promise<void> {
const selfTest = argv.includes("--self-test");
const hooksOnly = argv.includes("--hooks-only");
const trustedDir = flag("--trusted-dir");
- const base = { claude: flag("--claude") ?? "claude", model: flag("--model") ?? "opus", ...(trustedDir ? { trustedDir: resolve(trustedDir) } : {}) };
+ const base = { claude: flag("--claude") ?? "claude", model: flag("--model") ?? DEFAULT_MODEL, ...(trustedDir ? { trustedDir: resolve(trustedDir) } : {}) };
const version = spawnSync(base.claude, ["--version"], { encoding: "utf8" }).stdout?.trim() || "unknown version";
process.on("SIGINT", () => {
for (const vault of [...fixtures.keys()]) removeFixture(vault);
process.exit(130);
});
+ const modelsSeen = new Set<string>();
+ const run = async (steps: readonly Step[], breakage: Breakage, withMod: boolean): Promise<Verdict[]> => {
+ const r = await runSession(steps, breakage, { ...base, withMod });
+ for (const model of r.models) modelsSeen.add(model);
+ return r.verdicts;
+ };
const loading = trustedDir ? `found in the trusted folder ${resolve(trustedDir)}` : "loaded with --plugin-dir";
console.log(`Delivery gate · ${version} · ${base.model} · ${hooksOnly ? "settings hooks only" : `the mod ${loading}`}${selfTest ? " · SELF-TEST" : ""}\n`);
@@ -874,20 +928,22 @@ async function main(): Promise<void> {
const runs: Array<{ breakage: Breakage; verdicts: Verdict[] }> = [];
for (const breakage of ["cut", "startup-only"] as const) {
const verdicts: Verdict[] = [];
- for (const steps of plan(true)) verdicts.push(...(await runSession(steps, breakage, { ...base, withMod: true })));
+ for (const steps of plan(true)) verdicts.push(...(await run(steps, breakage, true)));
print(` [${breakage}]`, verdicts);
runs.push({ breakage, verdicts });
}
const { exitCode, message } = selfTestOutcome(runs);
console.log(`\n${message}`);
- process.exitCode = exitCode;
+ console.log(modelNote(base.model, [...modelsSeen]));
+ process.exitCode = ranAsAsked(base.model, [...modelsSeen]) ? exitCode : 2;
return;
}
const verdicts: Verdict[] = [];
- if (!hooksOnly) for (const steps of plan(true)) verdicts.push(...(await runSession(steps, "none", { ...base, withMod: true })));
- for (const steps of plan(false)) verdicts.push(...(await runSession(steps, "none", { ...base, withMod: false })));
+ if (!hooksOnly) for (const steps of plan(true)) verdicts.push(...(await run(steps, "none", true)));
+ for (const steps of plan(false)) verdicts.push(...(await run(steps, "none", false)));
print("", verdicts);
+ console.log(`\n${modelNote(base.model, [...modelsSeen])}`);
const invalid = verdicts.filter((v) => v.outcome === "INVALID").length;
const failed = verdicts.filter((v) => v.outcome === "FAIL").length;
if (invalid > 0) console.log(`\n${invalid} checkpoint(s) could not be judged.`);
@@ -903,7 +959,8 @@ async function main(): Promise<void> {
console.log(filed.status === 0 ? `\nFiled: ${filed.stdout.trim()}` : `\nCould not file the issue: ${filed.stderr.trim()}`);
}
// A FAIL needs nothing else to be judged: it wins over an INVALID elsewhere.
- process.exitCode = failed > 0 ? 1 : invalid > 0 ? 2 : 0;
+ // A run that can't show it ran the model asked for can't be judged either (#346).
+ process.exitCode = failed > 0 ? 1 : invalid > 0 || !ranAsAsked(base.model, [...modelsSeen]) ? 2 : 0;
}
if (isMainModule(import.meta.url)) {