Multiplayer agent harness for work.
fix(admin): restore Auto quarantine prompt controls (#1063)
Joshua Francefafbd6299d05
diff --git a/plugins/admin/public/index.html b/plugins/admin/public/index.htmlindex a6fd0a12a..3ab9d6cf3 100644--- a/plugins/admin/public/index.html+++ b/plugins/admin/public/index.html@@ -5281,6 +5281,52 @@><span class="status" id="st-security-posture"></span></div></section>+ <section class="card sv-governance hidden" id="card-auto-flagger">+ <div class="head">+ <h2>Auto flagger</h2>+ <p>+ Model and prompt that decide whether external context and tool results are quarantined in Auto+ mode. The security boundary and required verdict format remain fixed.+ </p>+ </div>+ <div class="body">+ <label for="auto-flagger-harness">Harness</label>+ <select id="auto-flagger-harness" style="max-width: 360px"></select>+ <label for="auto-flagger-model">Model</label>+ <select id="auto-flagger-model" style="max-width: 440px"></select>+ <label for="auto-flagger-rubric">Quarantine prompt (classification rubric)</label>+ <textarea id="auto-flagger-rubric" rows="10" maxlength="20000" style="width: 100%"></textarea>+ </div>+ <div class="foot">+ <button class="primary" data-save="auto-flagger">Apply</button>+ <button id="auto-flagger-test" type="button">Test</button>+ <label+ for="auto-flagger-window"+ style="display: inline-flex; align-items: center; gap: 6px; margin: 0; font-size: 13px"+ >on the last+ <input+ id="auto-flagger-window"+ type="number"+ min="1"+ max="500"+ step="1"+ value="100"+ style="width: 78px"+ />+ screenings</label+ >+ <label+ style="display: inline-flex; align-items: center; gap: 6px; margin: 0; font-size: 13px"+ title="Also replay the configuration in effect today, so the flag rates are comparable"+ ><input type="checkbox" id="auto-flagger-compare" /> compare with current</label+ >+ <button id="auto-flagger-reset" type="button">Restore default</button>+ <span class="status" id="st-auto-flagger"></span>+ </div>+ <div class="foot" id="auto-flagger-test-result" hidden>+ <span class="status" id="st-auto-flagger-test"></span>+ </div>+ </section><section class="card sv-governance setting-row hidden" id="card-approval-grant-modes"><div class="head"><h2>Approval grant options</h2>@@ -6243,6 +6289,7 @@content.insertBefore(anchor, $("card-service-credentials"));[$("card-security-posture"),+ $("card-auto-flagger"),governanceAmbient,$("card-external-slack"),$("card-command-policy"),@@ -7836,6 +7883,44 @@.querySelectorAll('[data-choice-for="security-posture"]').forEach((input) => (input.checked = input.value === $("security-posture").value));}+ const showAutoFlagger = scope.startsWith("org:") && "autoFlagger" in (r.data || {});+ $("card-auto-flagger").classList.toggle("hidden", !showAutoFlagger);+ if (scopeChanged) $("auto-flagger-test-result").hidden = true;+ if (showAutoFlagger) {+ const current = r.data.autoFlagger || r.data.autoFlaggerDefault;+ const harness = $("auto-flagger-harness");+ const model = $("auto-flagger-model");+ const harnessOptions = [...(r.data.harnessOptions || [])];+ if (!harnessOptions.includes(current.harnessId)) harnessOptions.push(current.harnessId);+ const modelsByHarness = r.data.modelsByHarness || {};+ harness.textContent = "";+ harnessOptions.forEach((id) => {+ const option = document.createElement("option");+ option.value = id;+ option.textContent = { pi: "Pi", opencode: "OpenCode", codex: "Codex", claude: "Claude Code" }[id] || id;+ harness.appendChild(option);+ });+ harness.value = current.harnessId;+ model.textContent = "";+ const syncAutoFlaggerModels = () => {+ const options = [...(modelsByHarness[harness.value] || [])];+ const prior = model.value || current.modelId;+ if (harness.value === current.harnessId && !options.some((entry) => entry.id === current.modelId)) {+ options.push({ id: current.modelId, name: current.modelId + " (configured; unavailable)" });+ }+ model.textContent = "";+ options.forEach((entry) => {+ const option = document.createElement("option");+ option.value = entry.id;+ option.textContent = entry.name + " (" + entry.id + ")";+ model.appendChild(option);+ });+ model.value = options.some((entry) => entry.id === prior) ? prior : (options[0] || {}).id || "";+ };+ harness.oninput = syncAutoFlaggerModels;+ syncAutoFlaggerModels();+ $("auto-flagger-rubric").value = current.rubric;+ }const showGrantModes = "approvalGrantModes" in (r.data || {});$("card-approval-grant-modes").classList.toggle("hidden", !showGrantModes);if (showGrantModes) {@@ -10468,6 +10553,12 @@};const SAVE = {"security-posture": () => ({ posture: $("security-posture").value }),+ "auto-flagger": () => ({+ harnessId: $("auto-flagger-harness").value,+ modelId: $("auto-flagger-model").value,+ rubric: $("auto-flagger-rubric").value,+ }),+"approval-grant-modes": () => ({session: $("approval-grant-session").checked,always: $("approval-grant-always").checked,@@ -10499,6 +10590,7 @@};const SAVE_ST = {"security-posture": "st-security-posture",+ "auto-flagger": "st-auto-flagger","approval-grant-modes": "st-approval-grant-modes","command-policy": "st-policy",soul: "st-soul",@@ -10515,6 +10607,7 @@branding: "st-branding",};const SAVE_RELOADS = new Set([+ "auto-flagger","soul","external-slack-participants","runtime",@@ -10525,6 +10618,25 @@"branding",]);const sectionSnapshots = new Map();+ function autoFlaggerTestSummary(data) {+ if (!data.sampled) return data.message || "No past screenings to replay yet.";+ const pct = (value) => (value == null ? "n/a" : (value * 100).toFixed(1) + "%");+ const parts = ["Flagged " + data.flagged + " of " + data.scored + " scored (" + pct(data.flagRate) + ")"];+ if (data.baseline) {+ parts.push(+ "current rubric flags " ++ pct(data.baseline.flagRate) ++ " · " ++ plural(data.baseline.changed, "verdict") ++ " changed",+ );+ }+ if (data.unscreened) parts.push(plural(data.unscreened, "sample") + " came back unscreened");+ if (data.errors) parts.push(plural(data.errors, "sample") + " errored");+ if (data.partial) parts.push("stopped at the time limit — partial result");+ parts.push("sampled " + data.sampled + " · " + Math.round(data.durationMs / 100) / 10 + "s");+ return parts.join(" · ");+ }function sectionButton(key) {return document.querySelector('[data-save="' + key + '"]');}@@ -10605,6 +10717,50 @@return Promise.resolve(true);}let governanceSaveSeq = 0;+ $("auto-flagger-test").onclick = async () => {+ const requestedScope = scope;+ const button = $("auto-flagger-test");+ $("auto-flagger-test-result").hidden = false;+ const windowField = $("auto-flagger-window");+ const requested = Number(windowField.value);+ if (!Number.isInteger(requested) || requested < 1 || requested > 500) {+ return setStatus("st-auto-flagger-test", "Window must be a whole number between 1 and 500.", "err", true);+ }+ setStatus(+ "st-auto-flagger-test",+ "Replaying the last " + requested + " screenings through this rubric…",+ "",+ true,+ );+ button.disabled = true;+ try {+ const r = await api("POST", "/api/scopes/" + encodeURIComponent(requestedScope) + "/auto-flagger/test", {+ window: requested,+ compare: $("auto-flagger-compare").checked,+ harnessId: $("auto-flagger-harness").value,+ modelId: $("auto-flagger-model").value,+ rubric: $("auto-flagger-rubric").value,+ });+ if (requestedScope !== scope) return;+ if (!r.ok) {+ return setStatus("st-auto-flagger-test", r.data?.message || "Test run failed.", "err", true);+ }+ setStatus("st-auto-flagger-test", autoFlaggerTestSummary(r.data), r.data.errors ? "warn" : "ok", true);+ } finally {+ button.disabled = false;+ }+ };+ $("auto-flagger-reset").onclick = async () => {+ const requestedScope = scope;+ const r = await api("PUT", "/api/scopes/" + encodeURIComponent(requestedScope) + "/auto-flagger", {+ reset: true,+ });+ if (requestedScope !== scope) return;+ if (!r.ok) return setStatus("st-auto-flagger", r.data?.message || "Reset failed.", "err");+ await loadScope();+ if (requestedScope === scope) setStatus("st-auto-flagger", "Default restored", "ok");+ };+document.querySelectorAll("[data-save]").forEach((btn) => {btn.onclick = async () => {const key = btn.dataset.save;diff --git a/plugins/admin/test/auto-flagger.test.ts b/plugins/admin/test/auto-flagger.test.tsnew file mode 100644index 000000000..7fc5294cc--- /dev/null+++ b/plugins/admin/test/auto-flagger.test.ts@@ -0,0 +1,207 @@+import assert from "node:assert/strict";+import { readFileSync } from "node:fs";+import test from "node:test";+import vm from "node:vm";++const html = readFileSync(new URL("../public/index.html", import.meta.url), "utf8");++test("org governance exposes the Auto quarantine rubric and replay controls", () => {+ assert.ok(html.includes('id="card-auto-flagger"'), "Auto flagger editor must exist");+ assert.match(html, /class="card sv-governance hidden" id="card-auto-flagger"/);+ for (const id of [+ "auto-flagger-harness",+ "auto-flagger-model",+ "auto-flagger-rubric",+ "auto-flagger-test",+ "auto-flagger-window",+ "auto-flagger-compare",+ "auto-flagger-reset",+ ]) {+ assert.ok(html.includes(`id="${id}"`), `${id} must exist`);+ }+ assert.match(html, /scope\.startsWith\("org:"\) && "autoFlagger" in/);+ assert.match(html, /r\.data\.autoFlagger \|\| r\.data\.autoFlaggerDefault/);+ assert.match(html, /"auto-flagger": \(\) => \(\{/);+ assert.match(html, /"auto-flagger": "st-auto-flagger"/);+ assert.match(html, /function autoFlaggerTestSummary\(data\)/);+});++function slice(from: string, to: string) {+ const start = html.indexOf(from);+ const end = html.indexOf(to, start);+ assert.ok(start >= 0 && end > start, `could not extract ${from}`);+ return html.slice(start, end);+}++class Element {+ tag: string;+ constructor(tag = "select") {+ this.tag = tag;+ }+ value = "";+ hidden = false;+ disabled = false;+ checked = false;+ text = "";+ options: Element[] = [];+ classes = new Set<string>();+ classList = {+ toggle: (name: string, on: boolean) => (on ? this.classes.add(name) : this.classes.delete(name)),+ };+ oninput?: () => void;+ onclick?: () => Promise<unknown>;+ set textContent(value: string) {+ this.text = value;+ this.options = [];+ if (this.tag === "select") this.value = "";+ }+ get textContent() {+ return this.text;+ }+ appendChild(option: Element) {+ this.options.push(option);+ }+}++function fixture(data: Record<string, unknown> = {}, scope = "org:example") {+ const elements: Record<string, Element> = {};+ const context = vm.createContext({+ scope,+ scopeChanged: true,+ r: { data },+ $: (id: string) => (elements[id] ??= new Element()),+ document: { createElement: (tag: string) => new Element(tag) },+ plural: (n: number, word: string) => `${n} ${word}${n === 1 ? "" : "s"}`,+ setStatus: (id: string, text: string) => ((elements[id] ??= new Element()).textContent = text),+ });+ return { elements, context };+}++const defaults = { harnessId: "pi", modelId: "model-a", rubric: "Default rubric" };+const loadData = {+ autoFlagger: null,+ autoFlaggerDefault: defaults,+ harnessOptions: ["pi", "claude"],+ modelsByHarness: {+ pi: [+ { id: "model-a", name: "A" },+ { id: "model-b", name: "B" },+ ],+ claude: [{ id: "model-c", name: "C" }],+ },+};+const loadSource = slice("const showAutoFlagger =", "const showGrantModes =");+const handlersSource = slice('$("auto-flagger-test").onclick =', 'document.querySelectorAll("[data-save]").forEach');+const summarySource = slice("function autoFlaggerTestSummary(data)", "function sectionButton(key)");++test("loading or restoring defaults replaces a previous model selection", () => {+ const { elements, context } = fixture(loadData);+ vm.runInContext(`{${loadSource}}`, context);+ assert.equal(elements["auto-flagger-model"].value, "model-a");+ elements["auto-flagger-model"].value = "model-b";+ vm.runInContext(`{${loadSource}}`, context);+ assert.equal(elements["auto-flagger-model"].value, "model-a");+ assert.equal(elements["auto-flagger-rubric"].value, "Default rubric");+ elements["auto-flagger-harness"].value = "claude";+ elements["auto-flagger-harness"].oninput!();+ assert.equal(elements["auto-flagger-model"].value, "model-c");+});++test("a saved flagger outside the current catalog is displayed, not silently replaced", () => {+ const { elements, context } = fixture({+ ...loadData,+ autoFlagger: { harnessId: "codex", modelId: "saved-model", rubric: "Saved rubric" },+ });+ vm.runInContext(`{${loadSource}}`, context);+ assert.equal(elements["auto-flagger-harness"].value, "codex");+ assert.ok(elements["auto-flagger-harness"].options.some((o) => o.value === "codex"));+ assert.equal(elements["auto-flagger-model"].value, "saved-model");+ assert.ok(elements["auto-flagger-model"].options.some((o) => o.value === "saved-model"));+ assert.equal(elements["auto-flagger-rubric"].value, "Saved rubric");+});++test("the card is org-only and follows the current governance layout", () => {+ for (const [scope, data] of [+ ["personal:example", loadData],+ ["org:example", {}],+ ] as const) {+ const { elements, context } = fixture(data, scope);+ vm.runInContext(`{${loadSource}}`, context);+ assert.ok(elements["card-auto-flagger"].classes.has("hidden"));+ }+ assert.match(html, /\$\("card-security-posture"\),\s*\$\("card-auto-flagger"\),\s*governanceAmbient/);+});++test("test sends an unsaved draft and comparison window without saving it", async () => {+ const { elements, context } = fixture(loadData);+ vm.runInContext(`{${loadSource}}\n${summarySource}\n${handlersSource}`, context);+ elements["auto-flagger-window"] = new Element();+ elements["auto-flagger-window"].value = "25";+ elements["auto-flagger-compare"] = new Element();+ elements["auto-flagger-compare"].checked = true;+ elements["auto-flagger-rubric"].value = "Unsaved draft — café";+ const calls: unknown[][] = [];+ context.api = async (...args: unknown[]) => {+ calls.push(args);+ return { ok: true, data: { sampled: 2, scored: 2, flagged: 1, flagRate: 0.5, durationMs: 100 } };+ };+ await elements["auto-flagger-test"].onclick!();+ assert.deepEqual(JSON.parse(JSON.stringify(calls)), [+ [+ "POST",+ "/api/scopes/org%3Aexample/auto-flagger/test",+ {+ window: 25,+ compare: true,+ harnessId: "pi",+ modelId: "model-a",+ rubric: "Unsaved draft — café",+ },+ ],+ ]);+ assert.match(elements["st-auto-flagger-test"].textContent, /Flagged 1 of 2 scored \(50.0%\)/);+ assert.equal(elements["auto-flagger-test"].disabled, false);+ elements["auto-flagger-window"].value = "0";+ await elements["auto-flagger-test"].onclick!();+ assert.equal(calls.length, 1);+ assert.equal(elements["auto-flagger-test-result"].hidden, false);+ assert.match(elements["st-auto-flagger-test"].textContent, /between 1 and 500/);+});++test("reset clears the override and does not reload a different scope after navigation", async () => {+ const { elements, context } = fixture(loadData);+ vm.runInContext(handlersSource, context);+ let reloads = 0;+ context.loadScope = async () => {+ reloads++;+ };+ context.api = async (method: string, path: string, body: unknown) => {+ assert.equal(method, "PUT");+ assert.equal(path, "/api/scopes/org%3Aexample/auto-flagger");+ assert.equal(JSON.stringify(body), '{"reset":true}');+ return { ok: true };+ };+ await elements["auto-flagger-reset"].onclick!();+ assert.equal(reloads, 1);+ assert.equal(elements["st-auto-flagger"].textContent, "Default restored");+ context.api = async () => {+ context.scope = "personal:other";+ return { ok: true };+ };+ await elements["auto-flagger-reset"].onclick!();+ assert.equal(reloads, 1);+});++test("replay summaries surface empty, partial, errored, and comparison results", () => {+ const { context } = fixture();+ vm.runInContext(summarySource, context);+ assert.match(vm.runInContext("autoFlaggerTestSummary({sampled: 0})", context), /No past screenings/);+ const summary = vm.runInContext(+ "autoFlaggerTestSummary({sampled:4,scored:2,flagged:1,flagRate:0.5,errors:1,unscreened:1,partial:true,durationMs:100,baseline:{flagRate:0,changed:1}})",+ context,+ );+ assert.match(summary, /current rubric flags 0.0%.*1 verdict changed/);+ assert.match(summary, /1 sample came back unscreened/);+ assert.match(summary, /1 sample errored/);+ assert.match(summary, /partial result/);+});