Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

Test a boundary rule against history before saving it (#241)

anygivenfriday3c1a067ef237
diff --git a/CHANGELOG.md b/CHANGELOG.md
index a7cced0e..43ed68cb 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -334,6 +334,33 @@ that matters is invisible.
The grant queries now wait for a surface to declare a real Bot, and simply do not run while the
placeholder holds. Conversation surfaces — the Bot page, channels — declare one and are unchanged.
+### A rule can be tested against history before it is saved
+
+A boundary was written blind: an administrator typed a CEL rule, saved it, and learned what it
+actually matches from the refusals it produced in production. The trail already records every judged
+computer action with the same facts the gateway judged it on, so the question "what would this rule
+have done" had an answer nobody could ask.
+
+The Boundaries page now has **Test first** beside **Add rule**. The candidate — the current policy
+plus the drafted rule — is replayed over recent recorded actions, and the reply names each one it
+would have decided differently and the rule that would have decided it. Nothing is saved and nothing
+is decided; no audit row is written, because no action was permitted or refused.
+
+Replay, not simulation: the context is rebuilt from the audit row exactly as the gateway built it at
+decision time, through the same helpers, so a rule behaves here as it will behave live. The scan is
+bounded and biased to recency, and the reply says how many rows it covered.
+
+### A browser refusal names the element again
+
+Every browser context carries a neutral all-empty `mcp` object, so a rule naming `mcp.effect`
+evaluates to false instead of throwing. The refusal copy keyed on that object being present rather
+than on its contents, so every live browser refusal took the tool-call branch and read
+": on is blocked" — two empty strings where the element and the page belonged. The tests passed,
+because their contexts omitted the field the gateway always attaches.
+
+The branch now keys on the server and tool being named, which a real tool call always has. A refused
+click reads "“Submit order” on shop.example is blocked by the rule ..." again, which is what the Bot
+relays to the person asking.
### Knowledge searches instead of guessing
diff --git a/app/src/lib/computers/queries.ts b/app/src/lib/computers/queries.ts
index 84cf9c09..ba632b5e 100644
--- a/app/src/lib/computers/queries.ts
+++ b/app/src/lib/computers/queries.ts
@@ -71,3 +71,44 @@ export function actionPolicyQueryOptions() {
}),
});
}
+
+/** One recorded action a candidate policy would decide differently than what happened. */
+export type DryRunChange = {
+ id: string;
+ createdAt: string;
+ action: string;
+ bot: string;
+ page: string;
+ element: { role: string; name: string } | null;
+ command: string | null;
+ file: string | null;
+ was: "allowed" | "refused";
+ would: "allowed" | "refused";
+ rule: string | null;
+ reason: string;
+};
+
+export type DryRunReport = {
+ scanned: number;
+ wouldRefuse: number;
+ wouldAllow: number;
+ unchanged: number;
+ /** Capped by the server; the counts cover everything scanned. */
+ changes: DryRunChange[];
+};
+
+/**
+ * What would this policy have decided, about recent recorded actions?
+ *
+ * A plain function rather than a query: the answer is about this candidate at this moment, nothing
+ * caches it and nothing invalidates it. It writes nothing — not the policy, and no audit row.
+ */
+export async function dryRunActionPolicy(
+ candidate: ActionPolicy,
+): Promise<DryRunReport> {
+ return client("/api/computers/policy-dry-run", "report", {
+ method: "POST",
+ body: { policy: candidate },
+ fallback: "The rule could not be tested against history.",
+ });
+}
diff --git a/app/src/routes/_authed/admin/boundaries.tsx b/app/src/routes/_authed/admin/boundaries.tsx
index cb6faca6..a4830562 100644
--- a/app/src/routes/_authed/admin/boundaries.tsx
+++ b/app/src/routes/_authed/admin/boundaries.tsx
@@ -6,6 +6,8 @@ import { saveActionPolicyMutationOptions } from "@/lib/computers/mutations";
import {
type ActionPolicy,
actionPolicyQueryOptions,
+ type DryRunReport,
+ dryRunActionPolicy,
type PolicyMode,
} from "@/lib/computers/queries";
import { queryClient } from "@/query-client";
@@ -51,6 +53,12 @@ function BoundariesPage() {
const [saved, setSaved] = useState(false);
const [draft, setDraft] = useState("");
+ const [tested, setTested] = useState<{
+ rule: string;
+ report: DryRunReport;
+ } | null>(null);
+ const [testing, setTesting] = useState(false);
+
const stored = useQuery(actionPolicyQueryOptions());
const savePolicy = useMutation(saveActionPolicyMutationOptions(queryClient));
@@ -90,6 +98,31 @@ function BoundariesPage() {
if (!trimmed || policy.deny.includes(trimmed)) return;
void save({ ...policy, deny: [...policy.deny, trimmed] });
setDraft("");
+ setTested(null);
+ };
+
+ /*
+ * The rule as it would be in force — the current policy plus this draft — replayed over recent
+ * recorded actions. Nothing is saved and nothing is decided; the reply names the actions the
+ * addition would have decided differently, so the rule's real reach is known before it starts
+ * refusing anybody.
+ */
+ const testRule = async (rule: string) => {
+ const trimmed = rule.trim();
+ if (!trimmed) return;
+ setProblem(null);
+ setTesting(true);
+ try {
+ const report = await dryRunActionPolicy({
+ ...policy,
+ deny: [...policy.deny, trimmed],
+ });
+ setTested({ rule: trimmed, report });
+ } catch (thrown) {
+ setProblem((thrown as Error).message);
+ } finally {
+ setTesting(false);
+ }
};
return (
@@ -191,6 +224,7 @@ function BoundariesPage() {
onChange={(event) => {
setDraft(event.target.value);
setSaved(false);
+ setTested(null);
}}
onKeyDown={(event) => {
if (event.key === "Enter") addRule(draft);
@@ -198,6 +232,14 @@ function BoundariesPage() {
placeholder='tool.name == "computer_click" && contains(element.name, "submit")'
value={draft}
/>
+ <Button
+ disabled={testing || draft.trim().length === 0}
+ onClick={() => void testRule(draft)}
+ size="sm"
+ variant="outline"
+ >
+ {testing ? "Testing…" : "Test first"}
+ </Button>
<Button
disabled={saving || draft.trim().length === 0}
onClick={() => addRule(draft)}
@@ -207,6 +249,8 @@ function BoundariesPage() {
</Button>
</div>
+ {tested ? <DryRunResult report={tested.report} /> : null}
+
<ul className="mt-3 space-y-2">
{PRESETS.map((preset) => (
<li className="flex items-start gap-3" key={preset.rule}>
@@ -256,3 +300,66 @@ function BoundariesPage() {
</PageShell>
);
}
+
+
+/**
+ * What the tested rule would have done to actions already on the trail.
+ *
+ * Says the number over everything scanned first, because the list below it is capped and a reader
+ * who stops at the rows should not believe the rows are the whole answer.
+ */
+function DryRunResult({ report }: { report: DryRunReport }) {
+ if (report.scanned === 0) {
+ return (
+ <p className="mt-2 text-xs text-muted-foreground" role="status">
+ No recorded computer actions to test against yet. The rule is valid;
+ what it matches will only be known once Bots have acted.
+ </p>
+ );
+ }
+
+ return (
+ <div className="mt-2" role="status">
+ <p className="text-xs text-muted-foreground">
+ {report.wouldRefuse === 0
+ ? `Tested against the last ${report.scanned} recorded actions: this rule would have refused none of them. It may still match future actions.`
+ : `Tested against the last ${report.scanned} recorded actions: this rule would have refused ${report.wouldRefuse}.`}
+ </p>
+ {report.changes.length > 0 ? (
+ <ul className="mt-2 divide-y divide-border rounded-md border border-border">
+ {report.changes.map((change) => (
+ <li className="px-3 py-2" key={change.id}>
+ <p className="text-xs">
+ <span className="font-medium">
+ {change.would === "refused"
+ ? "Would refuse"
+ : "Would now allow"}
+ </span>{" "}
+ <code className="font-mono">{change.action}</code>
+ {change.element?.name ? <> on “{change.element.name}”</> : null}
+ {change.command ? (
+ <>
+ {" "}
+ running <code className="font-mono">{change.command}</code>
+ </>
+ ) : null}
+ {change.file ? <> touching {change.file}</> : null}
+ </p>
+ <p className="mt-0.5 text-muted-foreground text-xs">
+ {change.bot}
+ {change.page ? <> · {change.page}</> : null} ·{" "}
+ {new Date(change.createdAt).toLocaleString()}
+ </p>
+ </li>
+ ))}
+ </ul>
+ ) : null}
+ {report.wouldRefuse > report.changes.length ? (
+ <p className="mt-1 text-muted-foreground text-xs">
+ Showing the first {report.changes.length}; the count above covers
+ everything scanned.
+ </p>
+ ) : null}
+ </div>
+ );
+}
diff --git a/server/src/app.ts b/server/src/app.ts
index 744a80c4..82e5a612 100644
--- a/server/src/app.ts
+++ b/server/src/app.ts
@@ -643,6 +643,7 @@ export function createApp(
requireUser,
canUseBot,
pageFrames,
+ auditReader,
),
);
}
diff --git a/server/src/computer/deployment-routes.ts b/server/src/computer/deployment-routes.ts
index a46daff2..678d7f42 100644
--- a/server/src/computer/deployment-routes.ts
+++ b/server/src/computer/deployment-routes.ts
@@ -11,4 +11,4 @@
* Nothing in the request distinguishes the two, so a Bot that really is called `policy` would be
* indistinguishable from the deployment route and would be served without the guard being asked.
*/
-export const DEPLOYMENT_ROUTES = new Set(["policy", "fleet"]);
+export const DEPLOYMENT_ROUTES = new Set(["policy", "fleet", "policy-dry-run"]);
diff --git a/server/src/computer/gateway.ts b/server/src/computer/gateway.ts
index b2fa548d..311c54ac 100644
--- a/server/src/computer/gateway.ts
+++ b/server/src/computer/gateway.ts
@@ -921,7 +921,7 @@ export function createComputerGateway(
* Lower-cased, because a rule forbidding `.env` must also catch `.ENV`; the
* operator should have anticipated. Same reasoning as the case-insensitive `contains` in policy.ts.
*/
-function describeFile(path: string): {
+export function describeFile(path: string): {
path: string;
name: string;
extension: string;
@@ -967,7 +967,7 @@ const ACTIVATING_KEYS = new Set(["Enter", "NumpadEnter", "Space", " "]);
*/
const HUMAN_GESTURES = new Set(["click", "type", "key", "scroll"]);
-function intentOf(
+export function intentOf(
toolName: string,
key: string | undefined,
): PolicyContext["intent"] {
@@ -1134,7 +1134,7 @@ async function writeControlEvent(
});
}
-function hostOf(url: string): string {
+export function hostOf(url: string): string {
try {
return new URL(url).host;
} catch {
diff --git a/server/src/computer/policy-dry-run.ts b/server/src/computer/policy-dry-run.ts
new file mode 100644
index 00000000..ba291d61
--- /dev/null
+++ b/server/src/computer/policy-dry-run.ts
@@ -0,0 +1,177 @@
+/**
+ * What would this policy have decided, about actions that already happened?
+ *
+ * A boundary is written blind: an administrator types a CEL rule, saves it, and finds out what it
+ * actually matches from the refusals it produces. The trail already holds everything needed to do
+ * better — every computer action is recorded with the same facts the gateway judged it on — so a
+ * candidate policy can be replayed over that history and answer, before it is saved, "these are the
+ * actions you would have decided differently."
+ *
+ * Replay, not simulation: the context handed to `evaluateActionPolicy` here is rebuilt from the
+ * audit row exactly as the gateway built it at decision time, through the same helpers. A rule that
+ * behaves one way here and another way live would make this feature worse than absent.
+ */
+
+import type { AuditEvent } from "../audit";
+import { describeFile, hostOf, intentOf } from "./gateway";
+import {
+ type ActionPolicy,
+ evaluateActionPolicy,
+ type PolicyContext,
+} from "./policy";
+
+/** The event types the gateway writes for a judged computer action. In one place, for the query. */
+export const REPLAYABLE_EVENT_TYPES = [
+ "computer.action_allowed",
+ "computer.action_refused",
+ "computer.action_failed",
+] as const;
+
+/** One action the candidate policy would have decided differently. */
+export type DryRunChange = {
+ id: string;
+ createdAt: string;
+ action: string;
+ bot: string;
+ page: string;
+ /** For a person reading the list; absent on file and command actions. */
+ element: { role: string; name: string } | null;
+ command: string | null;
+ file: string | null;
+ /** What actually happened, from the trail. A failed action was permitted first, so it was allowed. */
+ was: "allowed" | "refused";
+ would: "allowed" | "refused";
+ /** The candidate rule that decided it, or null for the default refusal. */
+ rule: string | null;
+ reason: string;
+};
+
+export type DryRunReport = {
+ /** Rows replayed. Bounded by what the caller asked the trail for, and says so. */
+ scanned: number;
+ wouldRefuse: number;
+ wouldAllow: number;
+ unchanged: number;
+ /** Capped; the counts above are over everything scanned. */
+ changes: DryRunChange[];
+};
+
+/** Changes returned in full detail. The counts still cover every scanned row. */
+const CHANGES_CAP = 50;
+
+/**
+ * The gateway's context, rebuilt from what it recorded.
+ *
+ * Field for field against the context the gateway constructs: absent browser facts become neutral
+ * empty strings rather than missing keys, because cel-js throws on an unbound identifier and a
+ * throw fails closed — a replay that refused everything the moment a rule named `command` would
+ * report a boundary far stricter than the one being proposed. `intent` is not stored on the row; it
+ * is derived from the tool and key here exactly as the gateway derives it at decision time.
+ *
+ * Null when the row does not carry enough to replay — a row from before a field existed, or a
+ * hand-inserted one. Skipped rather than guessed at.
+ */
+export function contextFromAuditPayload(
+ payload: Record<string, unknown>,
+): PolicyContext | null {
+ const action = payload.action;
+ const bot = payload.bot;
+ if (typeof action !== "string" || typeof bot !== "string") return null;
+
+ const text = (value: unknown): string =>
+ typeof value === "string" ? value : "";
+ const page = text(payload.page);
+ const key = text(payload.key);
+ const file = text(payload.file);
+
+ // Stored as an object on element actions, absent on file and command actions, and the literal
+ // sentence "not in the current snapshot" when the server could not identify the element. Only the
+ // object shape carries fields a rule can match; the other two replay as the neutral element.
+ const element =
+ payload.element && typeof payload.element === "object"
+ ? (payload.element as Record<string, unknown>)
+ : null;
+ const intent = intentOf(action, key || undefined);
+
+ return {
+ tool: { name: action },
+ bot: { id: bot },
+ actor: { id: text(payload.actor) },
+ page: { url: page, host: hostOf(page) },
+ ...(intent ? { intent } : {}),
+ key,
+ element: {
+ // The ref travels beside the element on the row, not inside it.
+ ref: text(payload.ref),
+ role: text(element?.role),
+ name: text(element?.name),
+ type: text(element?.type),
+ },
+ file: file ? describeFile(file) : { path: "", name: "", extension: "" },
+ command: text(payload.command),
+ mcp: { server: "", tool: "", effect: "" },
+ };
+}
+
+/**
+ * Replay judged actions under a candidate policy and report what changes.
+ *
+ * "Was" comes from the row's event type, which records what the policy in force decided — including
+ * a dry-run policy's refusals, which were recorded and then carried out. That is the honest
+ * baseline: the question this answers is "what would decide differently than was decided", not
+ * "what would run differently than ran".
+ */
+export function dryRunAgainstHistory(
+ policy: ActionPolicy,
+ events: AuditEvent[],
+): DryRunReport {
+ const report: DryRunReport = {
+ scanned: 0,
+ wouldRefuse: 0,
+ wouldAllow: 0,
+ unchanged: 0,
+ changes: [],
+ };
+
+ for (const event of events) {
+ const context = contextFromAuditPayload(event.payload);
+ if (!context) continue;
+ report.scanned += 1;
+
+ const was =
+ event.eventType === "computer.action_refused" ? "refused" : "allowed";
+ const decision = evaluateActionPolicy(policy, context);
+ const would = decision.allowed ? "allowed" : "refused";
+
+ if (was === would) {
+ report.unchanged += 1;
+ continue;
+ }
+ if (would === "refused") report.wouldRefuse += 1;
+ else report.wouldAllow += 1;
+
+ if (report.changes.length >= CHANGES_CAP) continue;
+ report.changes.push({
+ id: event.id,
+ createdAt: event.createdAt,
+ action: context.tool.name,
+ bot: context.bot.id,
+ page: context.page.url,
+ element:
+ context.element?.role || context.element?.name
+ ? {
+ role: context.element.role,
+ name: context.element.name,
+ }
+ : null,
+ command: context.command || null,
+ file: context.file?.path || null,
+ was,
+ would,
+ rule: decision.matched,
+ reason: decision.reason,
+ });
+ }
+
+ return report;
+}
diff --git a/server/src/computer/policy.ts b/server/src/computer/policy.ts
index bc9df1cc..1766ecc4 100644
--- a/server/src/computer/policy.ts
+++ b/server/src/computer/policy.ts
@@ -318,7 +318,12 @@ function describeRefusal(context: PolicyContext, expression: string): string {
);
}
- if (context.mcp) {
+ // Present is not enough: the gateway attaches a neutral all-empty `mcp` to every browser context
+ // so a rule naming `mcp.effect` evaluates to false instead of throwing. Testing the object rather
+ // than its contents made this branch fire for every browser refusal, and a person whose click was
+ // refused read ": on is blocked" — two empty strings where the element and page belonged. A real
+ // tool call always names its server and its tool, so those are what the branch keys on.
+ if (context.mcp?.server || context.mcp?.tool) {
return (
`This deployment's policy does not allow that: ${context.mcp.tool} on ` +
`${context.mcp.server} is blocked by the rule \`${expression}\`.`
diff --git a/server/src/computer/routes.ts b/server/src/computer/routes.ts
index e2e1dd74..d3fcb02b 100644
--- a/server/src/computer/routes.ts
+++ b/server/src/computer/routes.ts
@@ -17,7 +17,12 @@ import {
WorkspaceRequestError,
} from "./gateway";
import type { PageFrameStore } from "./page-frames";
+import type { AuditReader } from "../audit";
import { type PolicyStore, parseActionPolicy } from "./policy-store";
+import {
+ dryRunAgainstHistory,
+ REPLAYABLE_EVENT_TYPES,
+} from "./policy-dry-run";
/**
* The Bot computer's surface, behind the same session guard as every other API route.
@@ -41,6 +46,12 @@ export function createComputerRoutes(
canUseBot: BotAccessCheck,
/** Where the frame a page was opened on is kept. Absent leaves the transcript as it was. */
pageFrames?: PageFrameStore,
+ /**
+ * For the policy dry-run, which replays the trail. Optional the way `auditReader` is optional in
+ * `createApp`: a deployment wired without one still governs and records actions, and the dry-run
+ * endpoint says it cannot answer rather than answering from nothing.
+ */
+ auditReader?: AuditReader,
) {
const routes = new Hono<{ Variables: AppVariables }>();
@@ -613,6 +624,57 @@ export function createComputerRoutes(
return context.json({ policy: policyStore.get() });
});
+ /**
+ * What would this policy have decided, about actions already on the trail?
+ *
+ * A rule is otherwise written blind: saved first, understood later, from the refusals it produces
+ * in production. This answers before the save — the candidate is validated exactly as PUT
+ * validates it, replayed over recent judged actions, and the reply names each action it would have
+ * decided differently and the rule that would have decided it.
+ *
+ * A POST that writes nothing: not the policy, and no audit row either. Nothing is decided here —
+ * no action is permitted or refused, nothing runs or is stopped — and a trail row for every
+ * what-if would bury the rows that record what actually happened. Deployment-wide and named in
+ * DEPLOYMENT_ROUTES beside `/policy`, and admin-gated the same way, because history is the
+ * administrator's view.
+ */
+ routes.post("/policy-dry-run", requireUser, async (context) => {
+ const denied = requireAdmin(context);
+ if (denied) return denied;
+
+ if (!auditReader) {
+ return context.json(
+ {
+ error:
+ "This deployment records no readable trail, so there is no history to test against.",
+ },
+ 501,
+ );
+ }
+
+ const body = (await context.req.json().catch(() => null)) as {
+ policy?: unknown;
+ limit?: unknown;
+ } | null;
+ const parsed = parseActionPolicy(body?.policy);
+ if (!parsed.ok) {
+ return context.json({ error: parsed.error }, 400);
+ }
+
+ // Bounded, and biased to recency: the question is what this rule does to the traffic the
+ // deployment actually has, and last week's traffic answers that better than a full scan.
+ const requested = typeof body?.limit === "number" ? body.limit : 200;
+ const limit = Math.min(Math.max(Math.trunc(requested), 1), 500);
+
+ const { events } = await auditReader.list({
+ limit,
+ eventType: REPLAYABLE_EVENT_TYPES.join(","),
+ targetType: "computer",
+ });
+
+ return context.json({ report: dryRunAgainstHistory(parsed.policy, events) });
+ });
+
return routes;
}
diff --git a/server/tests/computer-policy.test.ts b/server/tests/computer-policy.test.ts
index 2f834927..60c2462d 100644
--- a/server/tests/computer-policy.test.ts
+++ b/server/tests/computer-policy.test.ts
@@ -619,3 +619,53 @@ describe("a rule about one surface does not refuse another", () => {
expect(decision.source).toBe("deny");
});
});
+
+
+describe("refusal wording under the context the gateway actually builds", () => {
+ /*
+ * The gateway attaches a neutral all-empty `mcp` to every browser context so a rule naming
+ * `mcp.effect` evaluates instead of throwing. The refusal copy used to key on the object being
+ * present rather than on its contents, so every live browser refusal read ": on is blocked" —
+ * empty strings where the element and page belonged — while tests that omitted the field passed.
+ */
+ test("a browser refusal names the element, neutral mcp notwithstanding", () => {
+ const decision = evaluateActionPolicy(
+ { mode: "enforce", deny: ['contains(element.name, "Submit")'], allow: ["true"] },
+ {
+ tool: { name: "computer_click" },
+ bot: { id: "general-assistant" },
+ actor: { id: "user:dev" },
+ page: { url: "https://example.com/checkout", host: "example.com" },
+ intent: "activate",
+ key: "",
+ element: { ref: "e1", role: "button", name: "Submit order", type: "" },
+ file: { path: "", name: "", extension: "" },
+ command: "",
+ mcp: { server: "", tool: "", effect: "" },
+ },
+ );
+ expect(decision.allowed).toBe(false);
+ expect(decision.reason).toContain("Submit order");
+ expect(decision.reason).toContain("example.com");
+ expect(decision.reason).not.toContain(" on is blocked");
+ });
+
+ test("a real tool call still reads as its server and tool", () => {
+ const decision = evaluateActionPolicy(
+ { mode: "enforce", deny: ['mcp.server == "notes"'], allow: ["true"] },
+ {
+ tool: { name: "mcp__notes__search_notes" },
+ bot: { id: "general-assistant" },
+ actor: { id: "user:dev" },
+ page: { url: "", host: "" },
+ key: "",
+ element: { ref: "", role: "", name: "", type: "" },
+ file: { path: "", name: "", extension: "" },
+ command: "",
+ mcp: { server: "notes", tool: "search_notes", effect: "read" },
+ },
+ );
+ expect(decision.allowed).toBe(false);
+ expect(decision.reason).toContain("search_notes on notes");
+ });
+});
diff --git a/server/tests/policy-dry-run.test.ts b/server/tests/policy-dry-run.test.ts
new file mode 100644
index 00000000..cb875363
--- /dev/null
+++ b/server/tests/policy-dry-run.test.ts
@@ -0,0 +1,171 @@
+import { describe, expect, test } from "bun:test";
+import type { AuditEvent } from "../src/audit";
+import {
+ contextFromAuditPayload,
+ dryRunAgainstHistory,
+} from "../src/computer/policy-dry-run";
+import type { ActionPolicy } from "../src/computer/policy";
+
+/**
+ * The replay must judge a recorded action exactly as the gateway judged it live. Every case here is
+ * a divergence that would make the feature lie: a field stored beside the element instead of inside
+ * it, an intent that is derived rather than stored, a row that predates a field.
+ */
+
+const PERMIT_EVERYTHING: ActionPolicy = {
+ mode: "enforce",
+ deny: [],
+ allow: ["true"],
+};
+
+function event(
+ overrides: Partial<AuditEvent> & { payload: Record<string, unknown> },
+): AuditEvent {
+ return {
+ id: overrides.id ?? "evt-1",
+ actorUserId: null,
+ eventType: overrides.eventType ?? "computer.action_allowed",
+ targetType: "computer",
+ targetId: "general-assistant",
+ createdAt: overrides.createdAt ?? "2026-08-01T00:00:00.000Z",
+ payload: overrides.payload,
+ };
+}
+
+const CLICK_SUBMIT = {
+ action: "computer_click",
+ bot: "general-assistant",
+ actor: "user:dev",
+ page: "https://shop.example/checkout",
+ ref: "e12",
+ element: { role: "button", name: "Submit order" },
+};
+
+describe("contextFromAuditPayload", () => {
+ test("rebuilds the element with the ref that is stored beside it", () => {
+ const context = contextFromAuditPayload(CLICK_SUBMIT);
+ expect(context?.element).toEqual({
+ ref: "e12",
+ role: "button",
+ name: "Submit order",
+ type: "",
+ });
+ expect(context?.page.host).toBe("shop.example");
+ });
+
+ test("derives intent the way the gateway does, including Enter as activation", () => {
+ expect(contextFromAuditPayload(CLICK_SUBMIT)?.intent).toBe("activate");
+ const enter = contextFromAuditPayload({
+ action: "computer_key",
+ bot: "b",
+ key: "Enter",
+ });
+ expect(enter?.intent).toBe("activate");
+ const letter = contextFromAuditPayload({
+ action: "computer_key",
+ bot: "b",
+ key: "a",
+ });
+ expect(letter?.intent).toBe("type");
+ });
+
+ test("an unidentifiable element replays as the neutral element, not a throw", () => {
+ // The gateway records the sentence "not in the current snapshot" for these rows.
+ const context = contextFromAuditPayload({
+ action: "computer_click",
+ bot: "b",
+ element: "not in the current snapshot",
+ });
+ expect(context?.element).toEqual({ ref: "", role: "", name: "", type: "" });
+ });
+
+ test("a row without the facts to replay is skipped, not guessed at", () => {
+ expect(contextFromAuditPayload({ bot: "b" })).toBeNull();
+ expect(contextFromAuditPayload({ action: 7, bot: "b" })).toBeNull();
+ });
+});
+
+describe("dryRunAgainstHistory", () => {
+ test("a new deny reports the allowed actions it would now refuse, with the rule", () => {
+ const candidate: ActionPolicy = {
+ mode: "enforce",
+ deny: ['contains(element.name, "Submit")'],
+ allow: ["true"],
+ };
+ const report = dryRunAgainstHistory(candidate, [
+ event({ id: "a", payload: CLICK_SUBMIT }),
+ event({
+ id: "b",
+ payload: { ...CLICK_SUBMIT, element: { role: "link", name: "Help" } },
+ }),
+ ]);
+ expect(report.scanned).toBe(2);
+ expect(report.wouldRefuse).toBe(1);
+ expect(report.unchanged).toBe(1);
+ expect(report.changes).toHaveLength(1);
+ expect(report.changes[0]?.id).toBe("a");
+ expect(report.changes[0]?.was).toBe("allowed");
+ expect(report.changes[0]?.would).toBe("refused");
+ expect(report.changes[0]?.rule).toBe('contains(element.name, "Submit")');
+ });
+
+ test("a loosened policy reports refusals it would now allow", () => {
+ const report = dryRunAgainstHistory(PERMIT_EVERYTHING, [
+ event({
+ id: "r",
+ eventType: "computer.action_refused",
+ payload: CLICK_SUBMIT,
+ }),
+ ]);
+ expect(report.wouldAllow).toBe(1);
+ expect(report.changes[0]?.was).toBe("refused");
+ expect(report.changes[0]?.would).toBe("allowed");
+ });
+
+ test("a failed action was permitted first, so it counts as allowed", () => {
+ const deny: ActionPolicy = {
+ mode: "enforce",
+ deny: ['tool.name == "computer_click"'],
+ allow: ["true"],
+ };
+ const report = dryRunAgainstHistory(deny, [
+ event({
+ id: "f",
+ eventType: "computer.action_failed",
+ payload: CLICK_SUBMIT,
+ }),
+ ]);
+ expect(report.wouldRefuse).toBe(1);
+ expect(report.changes[0]?.was).toBe("allowed");
+ });
+
+ test("a rule naming a command does not refuse a click, because absent facts are neutral", () => {
+ const candidate: ActionPolicy = {
+ mode: "enforce",
+ deny: ['contains(command, "rm -rf")'],
+ allow: ["true"],
+ };
+ const report = dryRunAgainstHistory(candidate, [
+ event({ payload: CLICK_SUBMIT }),
+ ]);
+ // The honest answer to "is this click running rm -rf" is no. A replay that failed closed here
+ // would report the boundary as far stricter than the one proposed.
+ expect(report.wouldRefuse).toBe(0);
+ expect(report.unchanged).toBe(1);
+ });
+
+ test("counts cover every scanned row even past the detail cap", () => {
+ const candidate: ActionPolicy = {
+ mode: "enforce",
+ deny: ["true"],
+ allow: ["true"],
+ };
+ const events = Array.from({ length: 60 }, (_, index) =>
+ event({ id: `evt-${index}`, payload: CLICK_SUBMIT }),
+ );
+ const report = dryRunAgainstHistory(candidate, events);
+ expect(report.scanned).toBe(60);
+ expect(report.wouldRefuse).toBe(60);
+ expect(report.changes).toHaveLength(50);
+ });
+});