Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.
Say a dry-run action was not enforced only when the policy refused it and it went ahead (#530)
Vaibhav Zopec34ebd184166
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 48eafbd0..d178b163 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -8,6 +8,15 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged.## Unreleased+### The Audit page says "not enforced" only under a dry-run refusal that went ahead++On a deployment in `dry-run`, the Audit page printed "dry-run: recorded, not enforced" under every+allowed action, because an allowed action is always carried out, and under a tool call content+inspection had refused, because that row copied `carriedOut: true` from the policy step before the+call was stopped. The line now appears only on a row the policy refused and dry-run let through,+which is the one case it describes. A tool call refused for carrying credential material is now+recorded with `carriedOut: false` in every mode. Rows already written keep their old value, and the+page reads them correctly either way.### The Routines page says when nothing is there to run themA routine needs a second process to fire it, and a deployment that never started one looked exactlydiff --git a/app/src/routes/_authed/admin/audit.tsx b/app/src/routes/_authed/admin/audit.tsxindex 6bd2cc12..db871eb3 100644--- a/app/src/routes/_authed/admin/audit.tsx+++ b/app/src/routes/_authed/admin/audit.tsx@@ -378,7 +378,9 @@ function Row({{decision.rule}</div>) : null}- {decision.mode === "dry-run" && decision.carriedOut ? (+ {decision.mode === "dry-run" &&+ decision.allowed === false &&+ decision.carriedOut ? (<div className="text-xs text-muted-foreground">dry-run: recorded, not enforced</div>diff --git a/app/tests/audit-dry-run-label.test.tsx b/app/tests/audit-dry-run-label.test.tsxnew file mode 100644index 00000000..b206fd33--- /dev/null+++ b/app/tests/audit-dry-run-label.test.tsx@@ -0,0 +1,135 @@+import { afterAll, afterEach, beforeAll, expect, test } from "bun:test";+import { GlobalRegistrator } from "@happy-dom/global-registrator";+import { QueryClient, QueryClientProvider } from "@tanstack/react-query";+import {+ createMemoryHistory,+ createRootRoute,+ createRouter,+ RouterProvider,+} from "@tanstack/react-router";+import { cleanup, render, within } from "@testing-library/react";+import { agentKeys } from "@/lib/agents/queries";+import { auditKeys } from "@/lib/audit/queries";+import { Route as AuditRoute } from "@/routes/_authed/admin/audit";++beforeAll(() => GlobalRegistrator.register());++afterEach(() => cleanup());++afterAll(() => GlobalRegistrator.unregister());++const DRY_RUN_LINE = "dry-run: recorded, not enforced";++function event(+ eventType: string,+ payload: Record<string, unknown>,+): Record<string, unknown> {+ const tool = eventType.startsWith("mcp.");+ return {+ id: "event-1",+ actorUserId: null,+ initiatorKind: "person",+ initiatorId: null,+ eventType,+ targetType: tool ? "mcp_tool" : "computer",+ targetId: tool ? "google-drive/search_files" : "bot-a",+ payload: { bot: "bot-a", actor: "[email protected]", ...payload },+ createdAt: new Date(0).toISOString(),+ };+}++async function renderAuditRow(row: Record<string, unknown>) {+ const queryClient = new QueryClient({+ defaultOptions: {+ queries: { retry: false, staleTime: Number.POSITIVE_INFINITY },+ },+ });+ queryClient.setQueryData([...auditKeys.all, ""], { events: [row] });+ queryClient.setQueryData(agentKeys.list(false), []);+ const rootRoute = createRootRoute({+ component: AuditRoute.options.component,+ });+ const router = createRouter({+ routeTree: rootRoute,+ history: createMemoryHistory({ initialEntries: ["/"] }),+ });+ const view = render(+ <QueryClientProvider client={queryClient}>+ <RouterProvider router={router} />+ </QueryClientProvider>,+ );+ await view.findByText("Decision");+ return within(view.container.querySelector("tbody") as HTMLElement);+}++test("a dry-run refusal that went ahead says it was not enforced", async () => {+ const row = await renderAuditRow(+ event("computer.action_refused", {+ action: "computer_click",+ decision: {+ allowed: false,+ mode: "dry-run",+ source: "deny",+ rule: 'element.name == "Submit"',+ carriedOut: true,+ },+ }),+ );+ expect(row.queryByText(DRY_RUN_LINE)).not.toBeNull();+});++test("an action a dry-run policy allowed has nothing to not enforce", async () => {+ const row = await renderAuditRow(+ event("computer.action_allowed", {+ action: "computer_click",+ decision: {+ allowed: true,+ mode: "dry-run",+ source: "allow",+ rule: "true",+ carriedOut: true,+ },+ }),+ );+ expect(row.queryByText("Allowed")).not.toBeNull();+ expect(row.queryByText(DRY_RUN_LINE)).toBeNull();+});++test("a tool call content inspection stopped is not described as not enforced", async () => {+ const row = await renderAuditRow(+ event("mcp.call_rejected", {+ server: "google-drive",+ tool: "search_files",+ refusal: "sensitive_tool_arguments",+ contentInspection: {+ reason: "sensitive_content",+ findings: [{ category: "credential_field", path: "$.nested.apiKey" }],+ },+ decision: {+ allowed: true,+ mode: "dry-run",+ source: "allow",+ rule: "true",+ carriedOut: true,+ },+ }),+ );+ expect(row.queryByText("Blocked")).not.toBeNull();+ expect(row.queryByText(DRY_RUN_LINE)).toBeNull();+});++test("an enforced refusal has no dry-run line", async () => {+ const row = await renderAuditRow(+ event("computer.action_refused", {+ action: "computer_click",+ decision: {+ allowed: false,+ mode: "enforce",+ source: "deny",+ rule: 'element.name == "Submit"',+ carriedOut: false,+ },+ }),+ );+ expect(row.queryByText(DRY_RUN_LINE)).toBeNull();+});diff --git a/server/src/plugins/store.ts b/server/src/plugins/store.tsindex 852d82c9..e58cd852 100644--- a/server/src/plugins/store.ts+++ b/server/src/plugins/store.ts@@ -3044,6 +3044,7 @@ export function createPluginStore(options: PluginStoreOptions) {...(input.initiator ? { initiator: input.initiator } : {}),payload: {...decided,+ decision: { ...decided.decision, carriedOut: false },refusal: "sensitive_tool_arguments",contentInspection: {reason: contentDecision.reason,diff --git a/server/tests/plugin-store.integration.test.ts b/server/tests/plugin-store.integration.test.tsindex f9f920c5..d8b5b1b2 100644--- a/server/tests/plugin-store.integration.test.ts+++ b/server/tests/plugin-store.integration.test.ts@@ -394,6 +394,7 @@ describe("the policy is asked as well as the grant", () => {expect(rejected).toHaveLength(1);expect(rejected[0].payload).toMatchObject({bot: holderId,+ decision: { carriedOut: false },contentInspection: {reason: "sensitive_content",findings: [{ category: "credential_field", path: "$.nested.apiKey" }],