Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

fix(admin): restore Auto quarantine prompt controls (#1063)

Joshua Francefafbd6299d05
diff --git a/plugins/admin/public/index.html b/plugins/admin/public/index.html
index a6fd0a12a..3ab9d6cf3 100644
--- a/plugins/admin/public/index.html
+++ b/plugins/admin/public/index.html
@@ -5281,6 +5281,52 @@
><span class="status" id="st-security-posture"></span>
</div>
</section>
+ <section class="card sv-governance hidden" id="card-auto-flagger">
+ <div class="head">
+ <h2>Auto flagger</h2>
+ <p>
+ Model and prompt that decide whether external context and tool results are quarantined in Auto
+ mode. The security boundary and required verdict format remain fixed.
+ </p>
+ </div>
+ <div class="body">
+ <label for="auto-flagger-harness">Harness</label>
+ <select id="auto-flagger-harness" style="max-width: 360px"></select>
+ <label for="auto-flagger-model">Model</label>
+ <select id="auto-flagger-model" style="max-width: 440px"></select>
+ <label for="auto-flagger-rubric">Quarantine prompt (classification rubric)</label>
+ <textarea id="auto-flagger-rubric" rows="10" maxlength="20000" style="width: 100%"></textarea>
+ </div>
+ <div class="foot">
+ <button class="primary" data-save="auto-flagger">Apply</button>
+ <button id="auto-flagger-test" type="button">Test</button>
+ <label
+ for="auto-flagger-window"
+ style="display: inline-flex; align-items: center; gap: 6px; margin: 0; font-size: 13px"
+ >on the last
+ <input
+ id="auto-flagger-window"
+ type="number"
+ min="1"
+ max="500"
+ step="1"
+ value="100"
+ style="width: 78px"
+ />
+ screenings</label
+ >
+ <label
+ style="display: inline-flex; align-items: center; gap: 6px; margin: 0; font-size: 13px"
+ title="Also replay the configuration in effect today, so the flag rates are comparable"
+ ><input type="checkbox" id="auto-flagger-compare" /> compare with current</label
+ >
+ <button id="auto-flagger-reset" type="button">Restore default</button>
+ <span class="status" id="st-auto-flagger"></span>
+ </div>
+ <div class="foot" id="auto-flagger-test-result" hidden>
+ <span class="status" id="st-auto-flagger-test"></span>
+ </div>
+ </section>
<section class="card sv-governance setting-row hidden" id="card-approval-grant-modes">
<div class="head">
<h2>Approval grant options</h2>
@@ -6243,6 +6289,7 @@
content.insertBefore(anchor, $("card-service-credentials"));
[
$("card-security-posture"),
+ $("card-auto-flagger"),
governanceAmbient,
$("card-external-slack"),
$("card-command-policy"),
@@ -7836,6 +7883,44 @@
.querySelectorAll('[data-choice-for="security-posture"]')
.forEach((input) => (input.checked = input.value === $("security-posture").value));
}
+ const showAutoFlagger = scope.startsWith("org:") && "autoFlagger" in (r.data || {});
+ $("card-auto-flagger").classList.toggle("hidden", !showAutoFlagger);
+ if (scopeChanged) $("auto-flagger-test-result").hidden = true;
+ if (showAutoFlagger) {
+ const current = r.data.autoFlagger || r.data.autoFlaggerDefault;
+ const harness = $("auto-flagger-harness");
+ const model = $("auto-flagger-model");
+ const harnessOptions = [...(r.data.harnessOptions || [])];
+ if (!harnessOptions.includes(current.harnessId)) harnessOptions.push(current.harnessId);
+ const modelsByHarness = r.data.modelsByHarness || {};
+ harness.textContent = "";
+ harnessOptions.forEach((id) => {
+ const option = document.createElement("option");
+ option.value = id;
+ option.textContent = { pi: "Pi", opencode: "OpenCode", codex: "Codex", claude: "Claude Code" }[id] || id;
+ harness.appendChild(option);
+ });
+ harness.value = current.harnessId;
+ model.textContent = "";
+ const syncAutoFlaggerModels = () => {
+ const options = [...(modelsByHarness[harness.value] || [])];
+ const prior = model.value || current.modelId;
+ if (harness.value === current.harnessId && !options.some((entry) => entry.id === current.modelId)) {
+ options.push({ id: current.modelId, name: current.modelId + " (configured; unavailable)" });
+ }
+ model.textContent = "";
+ options.forEach((entry) => {
+ const option = document.createElement("option");
+ option.value = entry.id;
+ option.textContent = entry.name + " (" + entry.id + ")";
+ model.appendChild(option);
+ });
+ model.value = options.some((entry) => entry.id === prior) ? prior : (options[0] || {}).id || "";
+ };
+ harness.oninput = syncAutoFlaggerModels;
+ syncAutoFlaggerModels();
+ $("auto-flagger-rubric").value = current.rubric;
+ }
const showGrantModes = "approvalGrantModes" in (r.data || {});
$("card-approval-grant-modes").classList.toggle("hidden", !showGrantModes);
if (showGrantModes) {
@@ -10468,6 +10553,12 @@
};
const SAVE = {
"security-posture": () => ({ posture: $("security-posture").value }),
+ "auto-flagger": () => ({
+ harnessId: $("auto-flagger-harness").value,
+ modelId: $("auto-flagger-model").value,
+ rubric: $("auto-flagger-rubric").value,
+ }),
+
"approval-grant-modes": () => ({
session: $("approval-grant-session").checked,
always: $("approval-grant-always").checked,
@@ -10499,6 +10590,7 @@
};
const SAVE_ST = {
"security-posture": "st-security-posture",
+ "auto-flagger": "st-auto-flagger",
"approval-grant-modes": "st-approval-grant-modes",
"command-policy": "st-policy",
soul: "st-soul",
@@ -10515,6 +10607,7 @@
branding: "st-branding",
};
const SAVE_RELOADS = new Set([
+ "auto-flagger",
"soul",
"external-slack-participants",
"runtime",
@@ -10525,6 +10618,25 @@
"branding",
]);
const sectionSnapshots = new Map();
+ function autoFlaggerTestSummary(data) {
+ if (!data.sampled) return data.message || "No past screenings to replay yet.";
+ const pct = (value) => (value == null ? "n/a" : (value * 100).toFixed(1) + "%");
+ const parts = ["Flagged " + data.flagged + " of " + data.scored + " scored (" + pct(data.flagRate) + ")"];
+ if (data.baseline) {
+ parts.push(
+ "current rubric flags " +
+ pct(data.baseline.flagRate) +
+ " · " +
+ plural(data.baseline.changed, "verdict") +
+ " changed",
+ );
+ }
+ if (data.unscreened) parts.push(plural(data.unscreened, "sample") + " came back unscreened");
+ if (data.errors) parts.push(plural(data.errors, "sample") + " errored");
+ if (data.partial) parts.push("stopped at the time limit — partial result");
+ parts.push("sampled " + data.sampled + " · " + Math.round(data.durationMs / 100) / 10 + "s");
+ return parts.join(" · ");
+ }
function sectionButton(key) {
return document.querySelector('[data-save="' + key + '"]');
}
@@ -10605,6 +10717,50 @@
return Promise.resolve(true);
}
let governanceSaveSeq = 0;
+ $("auto-flagger-test").onclick = async () => {
+ const requestedScope = scope;
+ const button = $("auto-flagger-test");
+ $("auto-flagger-test-result").hidden = false;
+ const windowField = $("auto-flagger-window");
+ const requested = Number(windowField.value);
+ if (!Number.isInteger(requested) || requested < 1 || requested > 500) {
+ return setStatus("st-auto-flagger-test", "Window must be a whole number between 1 and 500.", "err", true);
+ }
+ setStatus(
+ "st-auto-flagger-test",
+ "Replaying the last " + requested + " screenings through this rubric…",
+ "",
+ true,
+ );
+ button.disabled = true;
+ try {
+ const r = await api("POST", "/api/scopes/" + encodeURIComponent(requestedScope) + "/auto-flagger/test", {
+ window: requested,
+ compare: $("auto-flagger-compare").checked,
+ harnessId: $("auto-flagger-harness").value,
+ modelId: $("auto-flagger-model").value,
+ rubric: $("auto-flagger-rubric").value,
+ });
+ if (requestedScope !== scope) return;
+ if (!r.ok) {
+ return setStatus("st-auto-flagger-test", r.data?.message || "Test run failed.", "err", true);
+ }
+ setStatus("st-auto-flagger-test", autoFlaggerTestSummary(r.data), r.data.errors ? "warn" : "ok", true);
+ } finally {
+ button.disabled = false;
+ }
+ };
+ $("auto-flagger-reset").onclick = async () => {
+ const requestedScope = scope;
+ const r = await api("PUT", "/api/scopes/" + encodeURIComponent(requestedScope) + "/auto-flagger", {
+ reset: true,
+ });
+ if (requestedScope !== scope) return;
+ if (!r.ok) return setStatus("st-auto-flagger", r.data?.message || "Reset failed.", "err");
+ await loadScope();
+ if (requestedScope === scope) setStatus("st-auto-flagger", "Default restored", "ok");
+ };
+
document.querySelectorAll("[data-save]").forEach((btn) => {
btn.onclick = async () => {
const key = btn.dataset.save;
diff --git a/plugins/admin/test/auto-flagger.test.ts b/plugins/admin/test/auto-flagger.test.ts
new file mode 100644
index 000000000..7fc5294cc
--- /dev/null
+++ b/plugins/admin/test/auto-flagger.test.ts
@@ -0,0 +1,207 @@
+import assert from "node:assert/strict";
+import { readFileSync } from "node:fs";
+import test from "node:test";
+import vm from "node:vm";
+
+const html = readFileSync(new URL("../public/index.html", import.meta.url), "utf8");
+
+test("org governance exposes the Auto quarantine rubric and replay controls", () => {
+ assert.ok(html.includes('id="card-auto-flagger"'), "Auto flagger editor must exist");
+ assert.match(html, /class="card sv-governance hidden" id="card-auto-flagger"/);
+ for (const id of [
+ "auto-flagger-harness",
+ "auto-flagger-model",
+ "auto-flagger-rubric",
+ "auto-flagger-test",
+ "auto-flagger-window",
+ "auto-flagger-compare",
+ "auto-flagger-reset",
+ ]) {
+ assert.ok(html.includes(`id="${id}"`), `${id} must exist`);
+ }
+ assert.match(html, /scope\.startsWith\("org:"\) && "autoFlagger" in/);
+ assert.match(html, /r\.data\.autoFlagger \|\| r\.data\.autoFlaggerDefault/);
+ assert.match(html, /"auto-flagger": \(\) => \(\{/);
+ assert.match(html, /"auto-flagger": "st-auto-flagger"/);
+ assert.match(html, /function autoFlaggerTestSummary\(data\)/);
+});
+
+function slice(from: string, to: string) {
+ const start = html.indexOf(from);
+ const end = html.indexOf(to, start);
+ assert.ok(start >= 0 && end > start, `could not extract ${from}`);
+ return html.slice(start, end);
+}
+
+class Element {
+ tag: string;
+ constructor(tag = "select") {
+ this.tag = tag;
+ }
+ value = "";
+ hidden = false;
+ disabled = false;
+ checked = false;
+ text = "";
+ options: Element[] = [];
+ classes = new Set<string>();
+ classList = {
+ toggle: (name: string, on: boolean) => (on ? this.classes.add(name) : this.classes.delete(name)),
+ };
+ oninput?: () => void;
+ onclick?: () => Promise<unknown>;
+ set textContent(value: string) {
+ this.text = value;
+ this.options = [];
+ if (this.tag === "select") this.value = "";
+ }
+ get textContent() {
+ return this.text;
+ }
+ appendChild(option: Element) {
+ this.options.push(option);
+ }
+}
+
+function fixture(data: Record<string, unknown> = {}, scope = "org:example") {
+ const elements: Record<string, Element> = {};
+ const context = vm.createContext({
+ scope,
+ scopeChanged: true,
+ r: { data },
+ $: (id: string) => (elements[id] ??= new Element()),
+ document: { createElement: (tag: string) => new Element(tag) },
+ plural: (n: number, word: string) => `${n} ${word}${n === 1 ? "" : "s"}`,
+ setStatus: (id: string, text: string) => ((elements[id] ??= new Element()).textContent = text),
+ });
+ return { elements, context };
+}
+
+const defaults = { harnessId: "pi", modelId: "model-a", rubric: "Default rubric" };
+const loadData = {
+ autoFlagger: null,
+ autoFlaggerDefault: defaults,
+ harnessOptions: ["pi", "claude"],
+ modelsByHarness: {
+ pi: [
+ { id: "model-a", name: "A" },
+ { id: "model-b", name: "B" },
+ ],
+ claude: [{ id: "model-c", name: "C" }],
+ },
+};
+const loadSource = slice("const showAutoFlagger =", "const showGrantModes =");
+const handlersSource = slice('$("auto-flagger-test").onclick =', 'document.querySelectorAll("[data-save]").forEach');
+const summarySource = slice("function autoFlaggerTestSummary(data)", "function sectionButton(key)");
+
+test("loading or restoring defaults replaces a previous model selection", () => {
+ const { elements, context } = fixture(loadData);
+ vm.runInContext(`{${loadSource}}`, context);
+ assert.equal(elements["auto-flagger-model"].value, "model-a");
+ elements["auto-flagger-model"].value = "model-b";
+ vm.runInContext(`{${loadSource}}`, context);
+ assert.equal(elements["auto-flagger-model"].value, "model-a");
+ assert.equal(elements["auto-flagger-rubric"].value, "Default rubric");
+ elements["auto-flagger-harness"].value = "claude";
+ elements["auto-flagger-harness"].oninput!();
+ assert.equal(elements["auto-flagger-model"].value, "model-c");
+});
+
+test("a saved flagger outside the current catalog is displayed, not silently replaced", () => {
+ const { elements, context } = fixture({
+ ...loadData,
+ autoFlagger: { harnessId: "codex", modelId: "saved-model", rubric: "Saved rubric" },
+ });
+ vm.runInContext(`{${loadSource}}`, context);
+ assert.equal(elements["auto-flagger-harness"].value, "codex");
+ assert.ok(elements["auto-flagger-harness"].options.some((o) => o.value === "codex"));
+ assert.equal(elements["auto-flagger-model"].value, "saved-model");
+ assert.ok(elements["auto-flagger-model"].options.some((o) => o.value === "saved-model"));
+ assert.equal(elements["auto-flagger-rubric"].value, "Saved rubric");
+});
+
+test("the card is org-only and follows the current governance layout", () => {
+ for (const [scope, data] of [
+ ["personal:example", loadData],
+ ["org:example", {}],
+ ] as const) {
+ const { elements, context } = fixture(data, scope);
+ vm.runInContext(`{${loadSource}}`, context);
+ assert.ok(elements["card-auto-flagger"].classes.has("hidden"));
+ }
+ assert.match(html, /\$\("card-security-posture"\),\s*\$\("card-auto-flagger"\),\s*governanceAmbient/);
+});
+
+test("test sends an unsaved draft and comparison window without saving it", async () => {
+ const { elements, context } = fixture(loadData);
+ vm.runInContext(`{${loadSource}}\n${summarySource}\n${handlersSource}`, context);
+ elements["auto-flagger-window"] = new Element();
+ elements["auto-flagger-window"].value = "25";
+ elements["auto-flagger-compare"] = new Element();
+ elements["auto-flagger-compare"].checked = true;
+ elements["auto-flagger-rubric"].value = "Unsaved draft — café";
+ const calls: unknown[][] = [];
+ context.api = async (...args: unknown[]) => {
+ calls.push(args);
+ return { ok: true, data: { sampled: 2, scored: 2, flagged: 1, flagRate: 0.5, durationMs: 100 } };
+ };
+ await elements["auto-flagger-test"].onclick!();
+ assert.deepEqual(JSON.parse(JSON.stringify(calls)), [
+ [
+ "POST",
+ "/api/scopes/org%3Aexample/auto-flagger/test",
+ {
+ window: 25,
+ compare: true,
+ harnessId: "pi",
+ modelId: "model-a",
+ rubric: "Unsaved draft — café",
+ },
+ ],
+ ]);
+ assert.match(elements["st-auto-flagger-test"].textContent, /Flagged 1 of 2 scored \(50.0%\)/);
+ assert.equal(elements["auto-flagger-test"].disabled, false);
+ elements["auto-flagger-window"].value = "0";
+ await elements["auto-flagger-test"].onclick!();
+ assert.equal(calls.length, 1);
+ assert.equal(elements["auto-flagger-test-result"].hidden, false);
+ assert.match(elements["st-auto-flagger-test"].textContent, /between 1 and 500/);
+});
+
+test("reset clears the override and does not reload a different scope after navigation", async () => {
+ const { elements, context } = fixture(loadData);
+ vm.runInContext(handlersSource, context);
+ let reloads = 0;
+ context.loadScope = async () => {
+ reloads++;
+ };
+ context.api = async (method: string, path: string, body: unknown) => {
+ assert.equal(method, "PUT");
+ assert.equal(path, "/api/scopes/org%3Aexample/auto-flagger");
+ assert.equal(JSON.stringify(body), '{"reset":true}');
+ return { ok: true };
+ };
+ await elements["auto-flagger-reset"].onclick!();
+ assert.equal(reloads, 1);
+ assert.equal(elements["st-auto-flagger"].textContent, "Default restored");
+ context.api = async () => {
+ context.scope = "personal:other";
+ return { ok: true };
+ };
+ await elements["auto-flagger-reset"].onclick!();
+ assert.equal(reloads, 1);
+});
+
+test("replay summaries surface empty, partial, errored, and comparison results", () => {
+ const { context } = fixture();
+ vm.runInContext(summarySource, context);
+ assert.match(vm.runInContext("autoFlaggerTestSummary({sampled: 0})", context), /No past screenings/);
+ const summary = vm.runInContext(
+ "autoFlaggerTestSummary({sampled:4,scored:2,flagged:1,flagRate:0.5,errors:1,unscreened:1,partial:true,durationMs:100,baseline:{flagRate:0,changed:1}})",
+ context,
+ );
+ assert.match(summary, /current rubric flags 0.0%.*1 verdict changed/);
+ assert.match(summary, /1 sample came back unscreened/);
+ assert.match(summary, /1 sample errored/);
+ assert.match(summary, /partial result/);
+});