Skip to content

Open-source Grok Bot alternative for named AI teammates. Local-first desktop, web, and mobile clients with per-bot tools, instructions, and memory. Independent fork of T3 Code.

feat(plugins): add local Codex computer use (#68)

Leo966ed2fc8633
diff --git a/apps/server/src/mcp/PreviewSnapshotRedaction.ts b/apps/server/src/mcp/PreviewSnapshotRedaction.ts
index 31e3cf107..46dc69546 100644
--- a/apps/server/src/mcp/PreviewSnapshotRedaction.ts
+++ b/apps/server/src/mcp/PreviewSnapshotRedaction.ts
@@ -3,7 +3,7 @@ import { PNG } from "pngjs";
import { redactSensitiveText } from "./SensitiveDataRedaction.ts";
const REDACTED = "[REDACTED]";
-const MAX_SCREENSHOT_BYTES = 20 * 1_024 * 1_024;
+export const MAX_SCREENSHOT_BYTES = 20 * 1_024 * 1_024;
const MAX_SCREENSHOT_PIXELS = 16_000_000;
const secretField =
/^(?:api[_-]?key|access[_-]?token|refresh[_-]?token|token|secret|password|authorization|cookie|set-cookie|session|sessionId|clientSecret|awsSecretAccessKey)$/i;
diff --git a/apps/server/src/orchestration/Layers/ProjectionPipeline.ts b/apps/server/src/orchestration/Layers/ProjectionPipeline.ts
index 76f3811f7..6de74143a 100644
--- a/apps/server/src/orchestration/Layers/ProjectionPipeline.ts
+++ b/apps/server/src/orchestration/Layers/ProjectionPipeline.ts
@@ -1424,6 +1424,7 @@ const makeOrchestrationProjectionPipeline = Effect.fn("makeOrchestrationProjecti
providerName: event.payload.session.providerName,
providerInstanceId: event.payload.session.providerInstanceId ?? null,
runtimeMode: event.payload.session.runtimeMode,
+ mcpServerIds: [...(event.payload.session.mcpServerIds ?? [])],
activeTurnId: event.payload.session.activeTurnId,
lastError: event.payload.session.lastError,
updatedAt: event.payload.session.updatedAt,
diff --git a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.test.ts b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.test.ts
index 3db74f6f4..0b1411f90 100644
--- a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.test.ts
+++ b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.test.ts
@@ -391,6 +391,7 @@ projectionSnapshotLayer("ProjectionSnapshotQuery", (it) => {
status: "running",
providerName: "codex",
runtimeMode: "approval-required",
+ mcpServerIds: [],
activeTurnId: asTurnId("turn-1"),
lastError: null,
updatedAt: "2026-02-24T00:00:07.000Z",
@@ -477,6 +478,7 @@ projectionSnapshotLayer("ProjectionSnapshotQuery", (it) => {
status: "running",
providerName: "codex",
runtimeMode: "approval-required",
+ mcpServerIds: [],
activeTurnId: asTurnId("turn-1"),
lastError: null,
updatedAt: "2026-02-24T00:00:07.000Z",
diff --git a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts
index fbb643060..63235fc3a 100644
--- a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts
+++ b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts
@@ -127,7 +127,9 @@ const ProjectionThreadActivityDbRowSchema = ProjectionThreadActivity.mapFields(
sequence: Schema.NullOr(NonNegativeInt),
}),
);
-const ProjectionThreadSessionDbRowSchema = ProjectionThreadSession;
+const ProjectionThreadSessionDbRowSchema = ProjectionThreadSession.mapFields(
+ Struct.assign({ mcpServerIds: Schema.fromJsonString(Schema.Array(McpServerId)) }),
+);
const ProjectionCheckpointDbRowSchema = ProjectionCheckpoint.mapFields(
Struct.assign({
files: Schema.fromJsonString(Schema.Array(OrchestrationCheckpointFile)),
@@ -342,6 +344,7 @@ function mapSessionRow(
providerName: row.providerName,
...(row.providerInstanceId !== null ? { providerInstanceId: row.providerInstanceId } : {}),
runtimeMode: row.runtimeMode,
+ mcpServerIds: row.mcpServerIds,
activeTurnId: row.activeTurnId,
lastError: row.lastError,
updatedAt: row.updatedAt,
@@ -722,6 +725,7 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () {
provider_session_id AS "providerSessionId",
provider_thread_id AS "providerThreadId",
runtime_mode AS "runtimeMode",
+ mcp_server_ids_json AS "mcpServerIds",
active_turn_id AS "activeTurnId",
last_error AS "lastError",
updated_at AS "updatedAt"
@@ -743,6 +747,7 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () {
sessions.provider_session_id AS "providerSessionId",
sessions.provider_thread_id AS "providerThreadId",
sessions.runtime_mode AS "runtimeMode",
+ sessions.mcp_server_ids_json AS "mcpServerIds",
sessions.active_turn_id AS "activeTurnId",
sessions.last_error AS "lastError",
sessions.updated_at AS "updatedAt"
@@ -768,6 +773,7 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () {
sessions.provider_session_id AS "providerSessionId",
sessions.provider_thread_id AS "providerThreadId",
sessions.runtime_mode AS "runtimeMode",
+ sessions.mcp_server_ids_json AS "mcpServerIds",
sessions.active_turn_id AS "activeTurnId",
sessions.last_error AS "lastError",
sessions.updated_at AS "updatedAt"
@@ -1194,6 +1200,7 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () {
provider_name AS "providerName",
provider_instance_id AS "providerInstanceId",
runtime_mode AS "runtimeMode",
+ mcp_server_ids_json AS "mcpServerIds",
active_turn_id AS "activeTurnId",
last_error AS "lastError",
updated_at AS "updatedAt"
@@ -1816,18 +1823,7 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () {
for (const row of sessionRows) {
updatedAt = maxIso(updatedAt, row.updatedAt);
- sessionsByThread.set(row.threadId, {
- threadId: row.threadId,
- status: row.status,
- providerName: row.providerName,
- ...(row.providerInstanceId !== null
- ? { providerInstanceId: row.providerInstanceId }
- : {}),
- runtimeMode: row.runtimeMode,
- activeTurnId: row.activeTurnId,
- lastError: row.lastError,
- updatedAt: row.updatedAt,
- });
+ sessionsByThread.set(row.threadId, mapSessionRow(row));
}
const repositoryIdentities = yield* resolveRepositoryIdentitiesForProjects(
diff --git a/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts b/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts
index c152a287e..1ec32bac3 100644
--- a/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts
+++ b/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts
@@ -683,6 +683,7 @@ const make = Effect.gen(function* () {
providerName: activeSession?.provider ?? preferredProvider,
providerInstanceId: activeSession?.providerInstanceId ?? desiredInstanceId,
runtimeMode: desiredRuntimeMode,
+ mcpServerIds: activeSession?.mcpServerIds ?? [],
activeTurnId: null,
lastError: null,
updatedAt: createdAt,
@@ -790,6 +791,7 @@ const make = Effect.gen(function* () {
providerName: session.provider,
providerInstanceId: session.providerInstanceId,
runtimeMode: desiredRuntimeMode,
+ mcpServerIds: session.mcpServerIds ?? [],
// Provider turn ids are not orchestration turn ids.
activeTurnId: null,
lastError: session.lastError ?? null,
@@ -1664,6 +1666,7 @@ const make = Effect.gen(function* () {
? { providerInstanceId: thread.session.providerInstanceId }
: {}),
runtimeMode: thread.session?.runtimeMode ?? DEFAULT_RUNTIME_MODE,
+ mcpServerIds: [],
activeTurnId: null,
lastError: thread.session?.lastError ?? null,
updatedAt: now,
diff --git a/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts b/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts
index 620e14f12..3d2e3a763 100644
--- a/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts
+++ b/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts
@@ -1774,6 +1774,7 @@ const make = Effect.gen(function* () {
? { providerInstanceId: event.providerInstanceId }
: {}),
runtimeMode: thread.session?.runtimeMode ?? "full-access",
+ mcpServerIds: thread.session?.mcpServerIds ?? [],
activeTurnId: nextActiveTurnId,
lastError,
updatedAt: now,
@@ -2024,6 +2025,7 @@ const make = Effect.gen(function* () {
? { providerInstanceId: event.providerInstanceId }
: {}),
runtimeMode: thread.session?.runtimeMode ?? "full-access",
+ mcpServerIds: thread.session?.mcpServerIds ?? [],
activeTurnId: eventTurnId ?? null,
lastError: runtimeErrorMessage,
updatedAt: now,
diff --git a/apps/server/src/persistence/Layers/ProjectionThreadSessions.ts b/apps/server/src/persistence/Layers/ProjectionThreadSessions.ts
index dcb750983..5853442de 100644
--- a/apps/server/src/persistence/Layers/ProjectionThreadSessions.ts
+++ b/apps/server/src/persistence/Layers/ProjectionThreadSessions.ts
@@ -2,6 +2,10 @@ import * as SqlClient from "effect/unstable/sql/SqlClient";
import * as SqlSchema from "effect/unstable/sql/SqlSchema";
import * as Effect from "effect/Effect";
import * as Layer from "effect/Layer";
+import * as Schema from "effect/Schema";
+import * as Struct from "effect/Struct";
+
+import { McpServerId } from "@t3tools/contracts";
import { toPersistenceSqlError } from "../Errors.ts";
@@ -13,6 +17,12 @@ import {
GetProjectionThreadSessionInput,
} from "../Services/ProjectionThreadSessions.ts";
+const ProjectionThreadSessionDbRow = ProjectionThreadSession.mapFields(
+ Struct.assign({
+ mcpServerIds: Schema.fromJsonString(Schema.Array(McpServerId)),
+ }),
+);
+
const makeProjectionThreadSessionRepository = Effect.gen(function* () {
const sql = yield* SqlClient.SqlClient;
@@ -26,6 +36,7 @@ const makeProjectionThreadSessionRepository = Effect.gen(function* () {
provider_name,
provider_instance_id,
runtime_mode,
+ mcp_server_ids_json,
active_turn_id,
last_error,
updated_at
@@ -36,6 +47,7 @@ const makeProjectionThreadSessionRepository = Effect.gen(function* () {
${row.providerName},
${row.providerInstanceId},
${row.runtimeMode},
+ ${JSON.stringify(row.mcpServerIds)},
${row.activeTurnId},
${row.lastError},
${row.updatedAt}
@@ -46,6 +58,7 @@ const makeProjectionThreadSessionRepository = Effect.gen(function* () {
provider_name = excluded.provider_name,
provider_instance_id = excluded.provider_instance_id,
runtime_mode = excluded.runtime_mode,
+ mcp_server_ids_json = excluded.mcp_server_ids_json,
active_turn_id = excluded.active_turn_id,
last_error = excluded.last_error,
updated_at = excluded.updated_at
@@ -54,7 +67,7 @@ const makeProjectionThreadSessionRepository = Effect.gen(function* () {
const getProjectionThreadSessionRow = SqlSchema.findOneOption({
Request: GetProjectionThreadSessionInput,
- Result: ProjectionThreadSession,
+ Result: ProjectionThreadSessionDbRow,
execute: ({ threadId }) =>
sql`
SELECT
@@ -63,6 +76,7 @@ const makeProjectionThreadSessionRepository = Effect.gen(function* () {
provider_name AS "providerName",
provider_instance_id AS "providerInstanceId",
runtime_mode AS "runtimeMode",
+ mcp_server_ids_json AS "mcpServerIds",
active_turn_id AS "activeTurnId",
last_error AS "lastError",
updated_at AS "updatedAt"
diff --git a/apps/server/src/persistence/Migrations.ts b/apps/server/src/persistence/Migrations.ts
index ca403b510..ebf57e646 100644
--- a/apps/server/src/persistence/Migrations.ts
+++ b/apps/server/src/persistence/Migrations.ts
@@ -71,6 +71,7 @@ import Migration0055 from "./Migrations/055_AkeruMemoryCandidates.ts";
import Migration0056 from "./Migrations/056_AkeruBotUsageLedger.ts";
import Migration0057 from "./Migrations/057_AkeruMemoryCandidateUpdates.ts";
import Migration0058 from "./Migrations/058_ProjectionDelegations.ts";
+import Migration0059 from "./Migrations/059_ProjectionThreadSessionMcpServers.ts";
/**
* Migration loader with all migrations defined inline.
@@ -141,6 +142,7 @@ export const migrationEntries = [
[56, "AkeruBotUsageLedger", Migration0056],
[57, "AkeruMemoryCandidateUpdates", Migration0057],
[58, "ProjectionDelegations", Migration0058],
+ [59, "ProjectionThreadSessionMcpServers", Migration0059],
] as const;
export const migrationManifest = migrationEntries.map(([id, name]) => [id, name] as const);
diff --git a/apps/server/src/persistence/Migrations/058_ProjectionDelegations.test.ts b/apps/server/src/persistence/Migrations/058_ProjectionDelegations.test.ts
index 0de251d2f..609958873 100644
--- a/apps/server/src/persistence/Migrations/058_ProjectionDelegations.test.ts
+++ b/apps/server/src/persistence/Migrations/058_ProjectionDelegations.test.ts
@@ -19,7 +19,9 @@ it.layer(Layer.mergeAll(NodeSqliteClient.layerMemory()))("058_ProjectionDelegati
assert.deepEqual(before, []);
yield* runMigrations();
- assert.deepEqual(migrationManifest.at(-1), [58, "ProjectionDelegations"]);
+ assert.isTrue(
+ migrationManifest.some(([id, name]) => id === 58 && name === "ProjectionDelegations"),
+ );
yield* sql`
INSERT INTO projection_delegations (delegation_id, record_json)
diff --git a/apps/server/src/persistence/Migrations/059_ProjectionThreadSessionMcpServers.test.ts b/apps/server/src/persistence/Migrations/059_ProjectionThreadSessionMcpServers.test.ts
new file mode 100644
index 000000000..67a14c478
--- /dev/null
+++ b/apps/server/src/persistence/Migrations/059_ProjectionThreadSessionMcpServers.test.ts
@@ -0,0 +1,29 @@
+import { assert, it } from "@effect/vitest";
+import * as Effect from "effect/Effect";
+import * as Layer from "effect/Layer";
+import * as SqlClient from "effect/unstable/sql/SqlClient";
+
+import { runMigrations } from "../Migrations.ts";
+import * as NodeSqliteClient from "../NodeSqliteClient.ts";
+
+it.layer(Layer.mergeAll(NodeSqliteClient.layerMemory()))(
+ "059_ProjectionThreadSessionMcpServers",
+ (it) => {
+ it.effect("adds an empty MCP server list to projected sessions", () =>
+ Effect.gen(function* () {
+ const sql = yield* SqlClient.SqlClient;
+ yield* runMigrations({ toMigrationInclusive: 57 });
+ yield* runMigrations({ toMigrationInclusive: 59 });
+
+ const columns = yield* sql<{
+ readonly name: string;
+ readonly notnull: number;
+ readonly dflt_value: string | null;
+ }>`PRAGMA table_info(projection_thread_sessions)`;
+ const mcpServerIds = columns.find((column) => column.name === "mcp_server_ids_json");
+ assert.equal(mcpServerIds?.notnull, 1);
+ assert.equal(mcpServerIds?.dflt_value, "'[]'");
+ }),
+ );
+ },
+);
diff --git a/apps/server/src/persistence/Migrations/059_ProjectionThreadSessionMcpServers.ts b/apps/server/src/persistence/Migrations/059_ProjectionThreadSessionMcpServers.ts
new file mode 100644
index 000000000..81bb2706c
--- /dev/null
+++ b/apps/server/src/persistence/Migrations/059_ProjectionThreadSessionMcpServers.ts
@@ -0,0 +1,15 @@
+import * as Effect from "effect/Effect";
+import * as SqlClient from "effect/unstable/sql/SqlClient";
+
+export default Effect.gen(function* () {
+ const sql = yield* SqlClient.SqlClient;
+ const columns = yield* sql<{ readonly name: string }>`
+ PRAGMA table_info(projection_thread_sessions)
+ `;
+ if (!columns.some((column) => column.name === "mcp_server_ids_json")) {
+ yield* sql`
+ ALTER TABLE projection_thread_sessions
+ ADD COLUMN mcp_server_ids_json TEXT NOT NULL DEFAULT '[]'
+ `;
+ }
+});
diff --git a/apps/server/src/persistence/Services/ProjectionThreadSessions.ts b/apps/server/src/persistence/Services/ProjectionThreadSessions.ts
index 0510d8ab3..4c0bcb17a 100644
--- a/apps/server/src/persistence/Services/ProjectionThreadSessions.ts
+++ b/apps/server/src/persistence/Services/ProjectionThreadSessions.ts
@@ -9,6 +9,7 @@
import {
RuntimeMode,
IsoDateTime,
+ McpServerId,
OrchestrationSessionStatus,
ProviderInstanceId,
ThreadId,
@@ -27,6 +28,7 @@ export const ProjectionThreadSession = Schema.Struct({
providerName: Schema.NullOr(Schema.String),
providerInstanceId: Schema.NullOr(ProviderInstanceId),
runtimeMode: RuntimeMode,
+ mcpServerIds: Schema.Array(McpServerId),
activeTurnId: Schema.NullOr(TurnId),
lastError: Schema.NullOr(Schema.String),
updatedAt: IsoDateTime,
diff --git a/apps/server/src/provider/AkeruMastraHarness.ts b/apps/server/src/provider/AkeruMastraHarness.ts
index 78f186559..f193e6cd5 100644
--- a/apps/server/src/provider/AkeruMastraHarness.ts
+++ b/apps/server/src/provider/AkeruMastraHarness.ts
@@ -40,6 +40,7 @@ import { AKERU_AGENT_INSTRUCTIONS, AKERU_BOT_INSTRUCTIONS } from "./AkeruAgentIn
import { akeruKimiProvider, type AkeruKimiAccess } from "./AkeruKimiProvider.ts";
import { createAkeruMastraTools } from "./AkeruMastraTools.ts";
import type { AkeruToolRuntime } from "./AkeruToolRuntime.ts";
+import { isCodexComputerUseTool } from "./CodexComputerUse.ts";
const DEFAULT_MODEL_ID = "openai/gpt-5.6-sol";
const decodeProductFeedbackToolDraft = Schema.decodeUnknownExit(ProductFeedbackToolDraft, {
@@ -363,7 +364,8 @@ function approvalAwareTools(
};
const existing = configured.needsApprovalFn ?? configured.requireApproval;
const needsApproval: NeedsApprovalFn = async (input, context) => {
- const protectedAction = akeruActionNeedsApproval(name, input);
+ const protectedAction =
+ isCodexComputerUseTool(name) || akeruActionNeedsApproval(name, input);
await options.syncThreadToolApproval?.(threadId, name, protectedAction);
return (
protectedAction ||
diff --git a/apps/server/src/provider/AkeruSessionResources.test.ts b/apps/server/src/provider/AkeruSessionResources.test.ts
index 53fdbaea2..714d62490 100644
--- a/apps/server/src/provider/AkeruSessionResources.test.ts
+++ b/apps/server/src/provider/AkeruSessionResources.test.ts
@@ -8,6 +8,7 @@ import { McpServerId } from "@t3tools/contracts";
import { afterEach, describe, expect, it, vi } from "vite-plus/test";
import { AkeruSessionResources } from "./AkeruSessionResources.ts";
+import { CODEX_COMPUTER_USE_SERVER_ID } from "./CodexComputerUse.ts";
import {
type AkeruBotWorkspace,
type AkeruRemoteSession,
@@ -50,6 +51,37 @@ function browser(overrides?: { reconnect?: () => Promise<void>; close?: () => Pr
};
}
+function computerServer() {
+ return {
+ id: CODEX_COMPUTER_USE_SERVER_ID as never,
+ name: "Computer Use",
+ transport: "stdio" as const,
+ command: "akeru-codex-computer-use",
+ enabled: true,
+ createdAt: "2026-01-01T00:00:00.000Z",
+ updatedAt: "2026-01-01T00:00:00.000Z",
+ };
+}
+
+function mcpManager(
+ status: { connected: boolean; toolCount: number; error?: string },
+ tools: Record<string, unknown> = {},
+) {
+ return {
+ init: vi.fn(async () => undefined),
+ disconnect: vi.fn(async () => undefined),
+ getTools: vi.fn(() => tools),
+ getServerStatuses: vi.fn(() => [
+ {
+ name: CODEX_COMPUTER_USE_SERVER_ID,
+ transport: "stdio" as const,
+ toolNames: status.toolCount > 0 ? [`${CODEX_COMPUTER_USE_SERVER_ID}_control`] : [],
+ ...status,
+ },
+ ]),
+ };
+}
+
const remoteInput = {
resourceScope: "shared",
workspaceResourceKey: "vercel:shared",
@@ -386,6 +418,67 @@ describe("AkeruSessionResources", () => {
await resources.shutdown();
});
+ it("allows one Computer Use controller and releases it on stop", async () => {
+ const manager = mcpManager({ connected: true, toolCount: 1 });
+ const resources = new AkeruSessionResources({
+ stateDir: stateDir(),
+ hostPlatform: "darwin",
+ makeRemoteWorkspace: async () => workspace(),
+ makeBotBrowser: () => browser(),
+ makeMcpManager: () => manager as never,
+ resolveComputerUseServer: async () => ({
+ command: "/local/launcher",
+ args: ["mcp"],
+ env: {},
+ }),
+ toMcpServerConfigs: () => ({
+ [CODEX_COMPUTER_USE_SERVER_ID]: { command: "sentinel" },
+ }),
+ });
+ const input = { ...remoteInput, mcpServers: [computerServer()] };
+
+ await resources.acquire({ ...input, threadId: "controller" });
+ await expect(resources.acquire({ ...input, threadId: "blocked" })).rejects.toThrow(
+ "already controlled",
+ );
+ await resources.release("controller");
+ await resources.acquire({ ...input, threadId: "replacement" });
+ await resources.shutdown();
+ });
+
+ it("redacts Computer Use results at the MCP tool boundary", async () => {
+ const execute = vi.fn(async () => ({ screenshot: { url: "https://example.com/frame.png" } }));
+ const toolName = `${CODEX_COMPUTER_USE_SERVER_ID}_control`;
+ const manager = mcpManager({ connected: true, toolCount: 1 }, { [toolName]: { execute } });
+ const resources = new AkeruSessionResources({
+ stateDir: stateDir(),
+ hostPlatform: "darwin",
+ makeRemoteWorkspace: async () => workspace(),
+ makeBotBrowser: () => browser(),
+ makeMcpManager: () => manager as never,
+ resolveComputerUseServer: async () => ({
+ command: "/local/launcher",
+ args: ["mcp"],
+ env: {},
+ }),
+ toMcpServerConfigs: () => ({
+ [CODEX_COMPUTER_USE_SERVER_ID]: { command: "sentinel" },
+ }),
+ });
+
+ await resources.acquire({
+ ...remoteInput,
+ threadId: "controller",
+ mcpServers: [computerServer()],
+ });
+ const tool = Reflect.get(resources.getConnectorTools("controller"), toolName) as {
+ execute: () => Promise<unknown>;
+ };
+ await expect(tool.execute()).rejects.toThrow("unknown screenshot");
+ expect(execute).toHaveBeenCalledOnce();
+ await resources.shutdown();
+ });
+
it("creates and attaches a browser for remote workspaces", async () => {
const browserEndpoint = vi.fn(async () => ({
url: "https://browser.example",
@@ -443,4 +536,67 @@ describe("AkeruSessionResources", () => {
expect(resources.getConnectorTools("remote-mcp")).toEqual({ exa_search: {} });
await resources.shutdown();
});
+
+ it("releases the Computer Use lock when MCP health fails", async () => {
+ const failed = mcpManager({ connected: false, toolCount: 0, error: "Accessibility denied" });
+ const healthy = mcpManager({ connected: true, toolCount: 1 });
+ const makeMcpManager = vi
+ .fn()
+ .mockReturnValueOnce(failed as never)
+ .mockReturnValueOnce(healthy as never);
+ const resources = new AkeruSessionResources({
+ stateDir: stateDir(),
+ hostPlatform: "darwin",
+ makeRemoteWorkspace: async () => workspace(),
+ makeBotBrowser: () => browser(),
+ makeMcpManager,
+ resolveComputerUseServer: async () => ({
+ command: "/local/launcher",
+ args: ["mcp"],
+ env: {},
+ }),
+ toMcpServerConfigs: () => ({
+ [CODEX_COMPUTER_USE_SERVER_ID]: { command: "sentinel" },
+ }),
+ });
+ const input = { ...remoteInput, mcpServers: [computerServer()] };
+
+ await expect(resources.acquire({ ...input, threadId: "failed" })).rejects.toThrow(
+ "needs Screen Recording and Accessibility permissions",
+ );
+ await resources.acquire({ ...input, threadId: "replacement" });
+ await resources.shutdown();
+ });
+
+ it("does not expose raw MCP startup errors", async () => {
+ const failed = mcpManager({
+ connected: false,
+ toolCount: 0,
+ error: "Failed at /private/tester/Secret App",
+ });
+ const resources = new AkeruSessionResources({
+ stateDir: stateDir(),
+ hostPlatform: "darwin",
+ makeRemoteWorkspace: async () => workspace(),
+ makeBotBrowser: () => browser(),
+ makeMcpManager: () => failed as never,
+ resolveComputerUseServer: async () => ({
+ command: "/local/launcher",
+ args: ["mcp"],
+ env: {},
+ }),
+ toMcpServerConfigs: () => ({
+ [CODEX_COMPUTER_USE_SERVER_ID]: { command: "sentinel" },
+ }),
+ });
+
+ await expect(
+ resources.acquire({
+ ...remoteInput,
+ threadId: "failed",
+ mcpServers: [computerServer()],
+ }),
+ ).rejects.toThrow("Computer Use MCP failed to start.");
+ await resources.shutdown();
+ });
});
diff --git a/apps/server/src/provider/AkeruSessionResources.ts b/apps/server/src/provider/AkeruSessionResources.ts
index 88253e910..e3805e664 100644
--- a/apps/server/src/provider/AkeruSessionResources.ts
+++ b/apps/server/src/provider/AkeruSessionResources.ts
@@ -1,4 +1,5 @@
// @effect-diagnostics nodeBuiltinImport:off
+import * as NodeFS from "node:fs";
import * as NodePath from "node:path";
import type { ToolsInput } from "@mastra/core/agent";
@@ -23,6 +24,13 @@ import {
type CreateRemoteBotWorkspaceInput,
} from "./botWorkspace.ts";
import { BotWorkspacePool, type BotWorkspaceLease } from "./botWorkspacePool.ts";
+import {
+ CODEX_COMPUTER_USE_SERVER_ID,
+ isCodexComputerUseServer,
+ isCodexComputerUseTool,
+ resolveCodexComputerUseServer,
+ sanitizeCodexComputerUseResult,
+} from "./CodexComputerUse.ts";
export interface AkeruSessionResourceInput {
readonly threadId: string;
@@ -46,6 +54,8 @@ export interface AkeruSessionResourcesOptions {
input: CreateRemoteBotWorkspaceInput,
) => Promise<AkeruBotWorkspace | Workspace>;
readonly makeBotBrowser?: (input: CreateBotBrowserInput) => BotBrowser;
+ readonly hostPlatform?: NodeJS.Platform;
+ readonly resolveComputerUseServer?: typeof resolveCodexComputerUseServer;
readonly onMcpServerConnectionFailure?: (serverId: McpServer["id"]) => void;
readonly toMcpServerConfigs: (
servers: readonly McpServer[],
@@ -66,6 +76,8 @@ export class AkeruSessionResources {
private readonly browserReferences = new Map<string, number>();
private readonly browserDestroyRequests = new Set<string>();
private readonly browserReconnects = new Map<string, Promise<void>>();
+ private readonly computerUseTemporaryDirectories = new Map<string, string>();
+ private controllingThreadId: string | undefined;
private shuttingDown = false;
constructor(options: AkeruSessionResourcesOptions) {
@@ -95,7 +107,18 @@ export class AkeruSessionResources {
private async acquireOnce(input: AkeruSessionResourceInput): Promise<AkeruSessionResourceView> {
const key = input.threadId;
+ const usesComputer = input.mcpServers.some((server) =>
+ isCodexComputerUseServer(String(server.id)),
+ );
try {
+ if (usesComputer) {
+ if (this.controllingThreadId && this.controllingThreadId !== key) {
+ throw new Error(
+ `Computer Use is already controlled by thread '${this.controllingThreadId}'. Stop it before starting another controller.`,
+ );
+ }
+ this.controllingThreadId = key;
+ }
const workspaceLease = await this.workspacePool.acquire(
input.workspaceResourceKey,
async () => {
@@ -182,10 +205,24 @@ export class AkeruSessionResources {
if (input.mcpServers.length > 0) {
const attachment = await browser.attachment();
+ const configs = this.options.toMcpServerConfigs(input.mcpServers, attachment);
+ if (usesComputer) {
+ if (!this.options.hostPlatform) {
+ throw new Error("Computer Use host platform is unavailable.");
+ }
+ const config = await (
+ this.options.resolveComputerUseServer ?? resolveCodexComputerUseServer
+ )({ platform: this.options.hostPlatform });
+ configs[CODEX_COMPUTER_USE_SERVER_ID] = config;
+ const temporaryDirectory = config.env?.TMPDIR;
+ if (typeof temporaryDirectory === "string") {
+ this.computerUseTemporaryDirectories.set(key, temporaryDirectory);
+ }
+ }
const manager = (this.options.makeMcpManager ?? createMcpManager)(
NodePath.join(this.options.stateDir, "bot-mcp-runtime"),
".akeru-runtime",
- this.options.toMcpServerConfigs(input.mcpServers, attachment),
+ configs,
);
this.mcpManagers.set(key, manager);
try {
@@ -194,6 +231,10 @@ export class AkeruSessionResources {
for (const server of input.mcpServers) {
this.options.onMcpServerConnectionFailure?.(server.id);
}
+ if (usesComputer) {
+ // eslint-disable-next-line preserve-caught-error -- MCP errors may contain private desktop paths.
+ throw new Error("Computer Use MCP failed to start.");
+ }
throw cause;
}
const serversById = new Map(input.mcpServers.map((server) => [String(server.id), server]));
@@ -202,6 +243,17 @@ export class AkeruSessionResources {
const server = serversById.get(status.name);
if (server) this.options.onMcpServerConnectionFailure?.(server.id);
}
+ if (usesComputer) {
+ const status = manager
+ .getServerStatuses()
+ .find((server) => server.name === CODEX_COMPUTER_USE_SERVER_ID);
+ if (!status?.connected || status.toolCount === 0) {
+ if (/screen recording|accessibility/i.test(status?.error ?? "")) {
+ throw new Error("Computer Use needs Screen Recording and Accessibility permissions.");
+ }
+ throw new Error("Computer Use MCP failed to start.");
+ }
+ }
}
return {
@@ -216,10 +268,31 @@ export class AkeruSessionResources {
}
getConnectorTools(threadId: string): ToolsInput {
- return {
+ const tools: ToolsInput = {
...this.threadBrowsers.get(threadId)?.tools,
...this.mcpManagers.get(threadId)?.getTools(),
};
+ return Object.fromEntries(
+ Object.entries(tools).map(([name, tool]) => {
+ const execute = Reflect.get(tool, "execute") as unknown;
+ if (!isCodexComputerUseTool(name) || typeof execute !== "function") {
+ return [name, tool];
+ }
+ return [
+ name,
+ {
+ ...tool,
+ execute: async (...args: readonly unknown[]) => {
+ const temporaryDirectory = this.computerUseTemporaryDirectories.get(threadId);
+ return sanitizeCodexComputerUseResult(
+ await Reflect.apply(execute, tool, args),
+ temporaryDirectory ? { temporaryDirectory } : undefined,
+ );
+ },
+ },
+ ];
+ }),
+ );
}
getMcpManager(threadId: string): McpManager | undefined {
@@ -246,6 +319,15 @@ export class AkeruSessionResources {
const manager = this.mcpManagers.get(threadId);
this.mcpManagers.delete(threadId);
if (manager) await manager.disconnect().catch((cause) => failures.push(cause));
+ const computerUseTemporaryDirectory = this.computerUseTemporaryDirectories.get(threadId);
+ this.computerUseTemporaryDirectories.delete(threadId);
+ if (computerUseTemporaryDirectory) {
+ try {
+ NodeFS.rmSync(computerUseTemporaryDirectory, { recursive: true, force: true });
+ } catch (cause) {
+ failures.push(cause);
+ }
+ }
const browser = this.threadBrowsers.get(threadId);
const resourceKey = this.browserResourceKeys.get(threadId);
@@ -281,6 +363,7 @@ export class AkeruSessionResources {
if (userComputerLease) {
await userComputerLease.release(options).catch((cause) => failures.push(cause));
}
+ if (this.controllingThreadId === threadId) this.controllingThreadId = undefined;
if (failures.length > 0) throw failures[0];
}
diff --git a/apps/server/src/provider/CodexComputerUse.test.ts b/apps/server/src/provider/CodexComputerUse.test.ts
new file mode 100644
index 000000000..942a1a386
--- /dev/null
+++ b/apps/server/src/provider/CodexComputerUse.test.ts
@@ -0,0 +1,222 @@
+// @effect-diagnostics nodeBuiltinImport:off
+import * as NodeFS from "node:fs";
+import * as NodeOS from "node:os";
+import * as NodePath from "node:path";
+import * as NodeURL from "node:url";
+
+import { PNG } from "pngjs";
+import { afterEach, describe, expect, it } from "vite-plus/test";
+
+import {
+ resolveCodexComputerUseServer,
+ sanitizeCodexComputerUseResult,
+} from "./CodexComputerUse.ts";
+
+const directories = new Set<string>();
+
+function plugin(command = "./bin/launcher") {
+ const root = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "akeru-computer-use-"));
+ directories.add(root);
+ NodeFS.mkdirSync(NodePath.join(root, "bin"));
+ NodeFS.writeFileSync(NodePath.join(root, "bin", "launcher"), "#!/bin/sh\n", { mode: 0o700 });
+ NodeFS.writeFileSync(
+ NodePath.join(root, ".mcp.json"),
+ JSON.stringify({
+ mcpServers: {
+ "computer-use": {
+ command,
+ args: ["mcp"],
+ cwd: ".",
+ env_vars: ["CODEX_HOME"],
+ },
+ },
+ }),
+ );
+ return root;
+}
+
+function listing(root: string, overrides?: Record<string, unknown>) {
+ return JSON.stringify({
+ installed: [
+ {
+ pluginId: "computer-use@openai-bundled",
+ installed: true,
+ enabled: true,
+ source: { source: "local", path: root },
+ ...overrides,
+ },
+ ],
+ available: [],
+ });
+}
+
+describe("Codex Computer Use resolver", () => {
+ afterEach(() => {
+ for (const directory of directories) {
+ NodeFS.rmSync(directory, { recursive: true, force: true });
+ }
+ directories.clear();
+ });
+
+ it("resolves the enabled official launcher without persisting its path", async () => {
+ const root = plugin();
+ const result = await resolveCodexComputerUseServer({
+ platform: "darwin",
+ run: async () => listing(root),
+ });
+
+ expect(result).toEqual({
+ command: NodeFS.realpathSync(NodePath.join(root, "bin", "launcher")),
+ args: ["mcp"],
+ env: {
+ HOME: NodeOS.homedir(),
+ ...(process.env.CODEX_HOME ? { CODEX_HOME: process.env.CODEX_HOME } : {}),
+ TMPDIR: expect.stringMatching(/akeru-computer-use-/),
+ },
+ });
+ if (typeof result.env?.TMPDIR === "string") directories.add(result.env.TMPDIR);
+ });
+
+ it("fails closed for unsupported, missing, disabled, and malformed plugins", async () => {
+ await expect(
+ resolveCodexComputerUseServer({ platform: "linux", run: async () => "" }),
+ ).rejects.toThrow("only on macOS");
+ await expect(
+ resolveCodexComputerUseServer({
+ platform: "darwin",
+ run: async () => JSON.stringify({ installed: [], available: [] }),
+ }),
+ ).rejects.toThrow("codex plugin add");
+ const root = plugin();
+ await expect(
+ resolveCodexComputerUseServer({
+ platform: "darwin",
+ run: async () => listing(root, { enabled: false }),
+ }),
+ ).rejects.toThrow("Enable");
+ await expect(
+ resolveCodexComputerUseServer({ platform: "darwin", run: async () => "not json" }),
+ ).rejects.toThrow("invalid plugin list");
+ });
+
+ it("rejects a launcher outside the plugin directory", async () => {
+ const root = plugin();
+ const external = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "akeru-external-"));
+ directories.add(external);
+ const externalLauncher = NodePath.join(external, "launcher");
+ NodeFS.writeFileSync(externalLauncher, "#!/bin/sh\n", { mode: 0o700 });
+ NodeFS.rmSync(NodePath.join(root, "bin", "launcher"));
+ NodeFS.symlinkSync(externalLauncher, NodePath.join(root, "bin", "launcher"));
+ await expect(
+ resolveCodexComputerUseServer({
+ platform: "darwin",
+ run: async () => listing(root),
+ }),
+ ).rejects.toThrow("escapes its plugin directory");
+ });
+
+ it("rejects a manifest outside the plugin directory", async () => {
+ const root = plugin();
+ const external = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "akeru-external-"));
+ directories.add(external);
+ const externalManifest = NodePath.join(external, ".mcp.json");
+ NodeFS.renameSync(NodePath.join(root, ".mcp.json"), externalManifest);
+ NodeFS.symlinkSync(externalManifest, NodePath.join(root, ".mcp.json"));
+
+ await expect(
+ resolveCodexComputerUseServer({
+ platform: "darwin",
+ run: async () => listing(root),
+ }),
+ ).rejects.toThrow("manifest escapes its plugin directory");
+ });
+
+ it("validates in-memory frames and removes temporary originals", () => {
+ const directory = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "akeru-frame-"));
+ directories.add(directory);
+ const path = NodePath.join(directory, "frame.png");
+ const png = new PNG({ width: 1, height: 1 });
+ png.data.fill(255);
+ NodeFS.writeFileSync(path, PNG.sync.write(png));
+
+ const result = sanitizeCodexComputerUseResult(
+ {
+ screenshot: { url: NodeURL.pathToFileURL(path).href },
+ },
+ { temporaryDirectory: directory },
+ );
+
+ expect(result).toMatchObject({
+ screenshot: { url: expect.stringMatching(/^data:image\/png/) },
+ });
+ const frameUrl = (result as { screenshot: { url: string } }).screenshot.url;
+ const frame = PNG.sync.read(
+ Buffer.from(frameUrl.slice("data:image/png;base64,".length), "base64"),
+ );
+ expect([...frame.data]).toEqual([0, 0, 0, 255]);
+ expect(NodeFS.existsSync(path)).toBe(false);
+
+ const inline = sanitizeCodexComputerUseResult({
+ type: "image",
+ mimeType: "image/png",
+ data: PNG.sync.write(png).toString("base64"),
+ }) as { data: string };
+ expect([...PNG.sync.read(Buffer.from(inline.data, "base64")).data]).toEqual([0, 0, 0, 255]);
+
+ const dataUrl = sanitizeCodexComputerUseResult({
+ screenshot: { url: `data:image/png;base64,${PNG.sync.write(png).toString("base64")}` },
+ }) as { screenshot: { url: string } };
+ expect([
+ ...PNG.sync.read(
+ Buffer.from(dataUrl.screenshot.url.slice("data:image/png;base64,".length), "base64"),
+ ).data,
+ ]).toEqual([0, 0, 0, 255]);
+
+ const missingPath = NodePath.join(directory, "private-window-title.png");
+ expect(() =>
+ sanitizeCodexComputerUseResult(
+ {
+ screenshot: { url: NodeURL.pathToFileURL(missingPath).href },
+ },
+ { temporaryDirectory: directory },
+ ),
+ ).toThrow("unreadable screenshot");
+ try {
+ sanitizeCodexComputerUseResult(
+ {
+ screenshot: { url: NodeURL.pathToFileURL(missingPath).href },
+ },
+ { temporaryDirectory: directory },
+ );
+ } catch (cause) {
+ expect(String(cause)).not.toContain(missingPath);
+ }
+
+ expect(() =>
+ sanitizeCodexComputerUseResult({
+ type: "image",
+ mimeType: "image/jpeg",
+ data: "invalid",
+ }),
+ ).toThrow("unsupported screenshot format");
+ expect(() =>
+ sanitizeCodexComputerUseResult({
+ type: "image",
+ mimeType: "image/png",
+ data: "A".repeat(28 * 1_024 * 1_024),
+ }),
+ ).toThrow("oversized screenshot");
+
+ const foreignDirectory = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "foreign-frame-"));
+ directories.add(foreignDirectory);
+ const foreignPath = NodePath.join(foreignDirectory, "frame.png");
+ NodeFS.writeFileSync(foreignPath, PNG.sync.write(png));
+ expect(() =>
+ sanitizeCodexComputerUseResult(
+ { screenshot: { url: NodeURL.pathToFileURL(foreignPath).href } },
+ { temporaryDirectory: directory },
+ ),
+ ).toThrow("outside its temporary directory");
+ expect(NodeFS.existsSync(foreignPath)).toBe(true);
+ });
+});
diff --git a/apps/server/src/provider/CodexComputerUse.ts b/apps/server/src/provider/CodexComputerUse.ts
new file mode 100644
index 000000000..90e807250
--- /dev/null
+++ b/apps/server/src/provider/CodexComputerUse.ts
@@ -0,0 +1,243 @@
+// @effect-diagnostics nodeBuiltinImport:off
+import * as NodeChildProcess from "node:child_process";
+import * as NodeFS from "node:fs";
+import * as NodeOS from "node:os";
+import * as NodePath from "node:path";
+import * as NodeURL from "node:url";
+import * as NodeUtil from "node:util";
+
+import type { McpServerConfig } from "@mastra/code-sdk/mcp/index";
+
+import { MAX_SCREENSHOT_BYTES, redactComputerScreenshot } from "../mcp/PreviewSnapshotRedaction.ts";
+
+export const CODEX_COMPUTER_USE_SERVER_ID = "builtin-computer-use";
+const CODEX_COMPUTER_USE_PLUGIN_ID = "computer-use@openai-bundled";
+const MAX_SCREENSHOT_BASE64_LENGTH = Math.ceil(MAX_SCREENSHOT_BYTES / 3) * 4;
+const execFile = NodeUtil.promisify(NodeChildProcess.execFile);
+
+type RunCodex = (args: readonly string[]) => Promise<string>;
+export interface CodexComputerUseServerConfig {
+ readonly command: string;
+ readonly args: string[];
+ readonly env: Record<string, string>;
+}
+
+function object(value: unknown, label: string): Record<string, unknown> {
+ if (typeof value !== "object" || value === null || Array.isArray(value)) {
+ throw new Error(`${label} is invalid.`);
+ }
+ return value as Record<string, unknown>;
+}
+
+function entries(value: unknown): readonly Record<string, unknown>[] {
+ if (!Array.isArray(value)) throw new Error("Codex returned an invalid plugin list.");
+ return value.map((entry) => object(entry, "Codex plugin entry"));
+}
+
+function string(value: unknown, label: string): string {
+ if (typeof value !== "string" || value.length === 0) throw new Error(`${label} is invalid.`);
+ return value;
+}
+
+async function runCodex(args: readonly string[]): Promise<string> {
+ const result = await execFile("codex", [...args], {
+ encoding: "utf8",
+ timeout: 10_000,
+ killSignal: "SIGKILL",
+ maxBuffer: 2 * 1_024 * 1_024,
+ });
+ return result.stdout;
+}
+
+export function isCodexComputerUseServer(id: string): boolean {
+ return id === CODEX_COMPUTER_USE_SERVER_ID;
+}
+
+export function isCodexComputerUseTool(name: string): boolean {
+ return name.startsWith(`${CODEX_COMPUTER_USE_SERVER_ID}_`);
+}
+
+function isWithin(parent: string, child: string): boolean {
+ const relative = NodePath.relative(parent, child);
+ return relative === "" || (!relative.startsWith("..") && !NodePath.isAbsolute(relative));
+}
+
+function screenshotDataUrl(url: string, temporaryDirectory?: string): string {
+ if (url.startsWith("data:image/png;base64,")) {
+ const encoded = url.slice("data:image/png;base64,".length);
+ if (encoded.length > MAX_SCREENSHOT_BASE64_LENGTH) {
+ throw new Error("Computer Use returned an oversized screenshot.");
+ }
+ const redacted = redactComputerScreenshot({
+ mediaType: "image/png",
+ data: Buffer.from(encoded, "base64"),
+ });
+ return `data:image/png;base64,${Buffer.from(redacted.data).toString("base64")}`;
+ }
+ if (!url.startsWith("file:")) throw new Error("Computer Use returned an unknown screenshot.");
+ if (!temporaryDirectory) throw new Error("Computer Use screenshot cleanup is unavailable.");
+ let path: string;
+ let ownedDirectory: string;
+ try {
+ path = NodeFS.realpathSync(NodeURL.fileURLToPath(url));
+ ownedDirectory = NodeFS.realpathSync(temporaryDirectory);
+ } catch {
+ throw new Error("Computer Use returned an unreadable screenshot.");
+ }
+ if (!isWithin(ownedDirectory, path)) {
+ throw new Error("Computer Use returned a screenshot outside its temporary directory.");
+ }
+ let bytes: Buffer | undefined;
+ try {
+ const redacted = redactComputerScreenshot({
+ mediaType: "image/png",
+ data: NodeFS.readFileSync(path),
+ });
+ bytes = Buffer.from(redacted.data);
+ } catch {
+ bytes = undefined;
+ }
+ try {
+ NodeFS.rmSync(path);
+ } catch {
+ throw new Error("Could not remove the temporary Computer Use screenshot.");
+ }
+ if (!bytes) throw new Error("Computer Use returned an invalid screenshot.");
+ return `data:image/png;base64,${bytes.toString("base64")}`;
+}
+
+export function sanitizeCodexComputerUseResult(
+ value: unknown,
+ options?: { readonly temporaryDirectory?: string },
+): unknown {
+ let remaining = 10_000;
+ const visit = (entry: unknown, field?: string): unknown => {
+ remaining -= 1;
+ if (remaining < 0) throw new Error("Computer Use returned too much data.");
+ if (Array.isArray(entry)) return entry.map((item) => visit(item));
+ if (typeof entry !== "object" || entry === null) {
+ if (field === "screenshot" && entry !== null) {
+ throw new Error("Computer Use returned an invalid screenshot.");
+ }
+ return entry;
+ }
+ const object = entry as Record<string, unknown>;
+ if (object.type === "image") {
+ const mediaType = object.mimeType ?? object.mediaType;
+ if (mediaType !== "image/png" || typeof object.data !== "string") {
+ throw new Error("Computer Use returned an unsupported screenshot format.");
+ }
+ if (object.data.length > MAX_SCREENSHOT_BASE64_LENGTH) {
+ throw new Error("Computer Use returned an oversized screenshot.");
+ }
+ const redacted = redactComputerScreenshot({
+ mediaType: "image/png",
+ data: Buffer.from(object.data, "base64"),
+ });
+ return { ...object, data: Buffer.from(redacted.data).toString("base64") };
+ }
+ if (field === "screenshot") {
+ if (typeof object.url !== "string") {
+ throw new Error("Computer Use returned an invalid screenshot.");
+ }
+ return { ...object, url: screenshotDataUrl(object.url, options?.temporaryDirectory) };
+ }
+ return Object.fromEntries(Object.entries(object).map(([key, item]) => [key, visit(item, key)]));
+ };
+ return visit(value);
+}
+
+export async function resolveCodexComputerUseServer(options: {
+ readonly platform: NodeJS.Platform;
+ readonly run?: RunCodex;
+}): Promise<CodexComputerUseServerConfig> {
+ if (options.platform !== "darwin") {
+ throw new Error("Codex Computer Use is supported only on macOS.");
+ }
+
+ let output: string;
+ try {
+ output = await (options.run ?? runCodex)([
+ "plugin",
+ "list",
+ "--marketplace",
+ "openai-bundled",
+ "--available",
+ "--json",
+ ]);
+ } catch {
+ throw new Error("Could not inspect the Codex Computer Use plugin.");
+ }
+
+ let document: Record<string, unknown>;
+ try {
+ document = object(JSON.parse(output), "Codex plugin list");
+ } catch {
+ throw new Error("Codex returned an invalid plugin list.");
+ }
+ const plugin = [...entries(document.installed), ...entries(document.available)].find(
+ (entry) => entry.pluginId === CODEX_COMPUTER_USE_PLUGIN_ID,
+ );
+ if (!plugin || plugin.installed !== true) {
+ throw new Error(
+ "Install the official Codex Computer Use plugin with `codex plugin add computer-use@openai-bundled --json`.",
+ );
+ }
+ if (plugin.enabled !== true) throw new Error("Enable the official Codex Computer Use plugin.");
+
+ const source = object(plugin.source, "Codex Computer Use plugin source");
+ if (source.source !== "local") throw new Error("Codex Computer Use must use a local plugin.");
+ let root: string;
+ let manifestPath: string;
+ let manifest: Record<string, unknown>;
+ try {
+ root = NodeFS.realpathSync(string(source.path, "Codex Computer Use plugin path"));
+ manifestPath = NodeFS.realpathSync(NodePath.join(root, ".mcp.json"));
+ if (!isWithin(root, manifestPath)) {
+ throw new Error("Computer Use MCP manifest escapes its plugin directory.");
+ }
+ manifest = object(JSON.parse(NodeFS.readFileSync(manifestPath, "utf8")), "MCP manifest");
+ } catch (cause) {
+ if (cause instanceof Error && cause.message.includes("escapes its plugin directory")) {
+ throw cause;
+ }
+ // eslint-disable-next-line preserve-caught-error -- Plugin paths must not reach persisted errors.
+ throw new Error("Could not read the Codex Computer Use plugin.");
+ }
+ const servers = object(manifest.mcpServers, "MCP servers");
+ const server = object(servers["computer-use"], "Computer Use MCP server");
+ const command = string(server.command, "Computer Use MCP command");
+ if (NodePath.isAbsolute(command) || server.cwd !== ".") {
+ throw new Error("Computer Use MCP must use its plugin-relative launcher.");
+ }
+ if (
+ !Array.isArray(server.args) ||
+ server.args.length !== 1 ||
+ server.args[0] !== "mcp" ||
+ !Array.isArray(server.env_vars) ||
+ server.env_vars.length !== 1 ||
+ server.env_vars[0] !== "CODEX_HOME"
+ ) {
+ throw new Error("Computer Use MCP has an unsupported launch configuration.");
+ }
+
+ let launcher: string;
+ try {
+ launcher = NodeFS.realpathSync(NodePath.resolve(root, command));
+ NodeFS.accessSync(launcher, NodeFS.constants.X_OK);
+ } catch {
+ throw new Error("Could not start the Codex Computer Use plugin.");
+ }
+ if (!isWithin(root, launcher)) {
+ throw new Error("Computer Use MCP launcher escapes its plugin directory.");
+ }
+ return {
+ command: launcher,
+ args: ["mcp"],
+ env: {
+ HOME: NodeOS.homedir(),
+ ...(process.env.CODEX_HOME ? { CODEX_HOME: process.env.CODEX_HOME } : {}),
+ TMPDIR: NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "akeru-computer-use-")),
+ },
+ } satisfies McpServerConfig;
+}
diff --git a/apps/server/src/provider/Layers/AgentController.test.ts b/apps/server/src/provider/Layers/AgentController.test.ts
index a8b6d2e5d..5fc2410ec 100644
--- a/apps/server/src/provider/Layers/AgentController.test.ts
+++ b/apps/server/src/provider/Layers/AgentController.test.ts
@@ -71,6 +71,38 @@ const codexSelection = {
instanceId: codexInstanceId,
model: "gpt-5.6-sol",
};
+const computerUseToolName = "builtin-computer-use_control";
+
+function computerUseServer() {
+ return {
+ id: McpServerId.make("builtin-computer-use"),
+ name: "Codex Computer Use",
+ transport: "stdio" as const,
+ command: "akeru-codex-computer-use",
+ enabled: true,
+ createdAt: "2026-01-01T00:00:00.000Z",
+ updatedAt: "2026-01-01T00:00:00.000Z",
+ };
+}
+
+function computerUseMcpManager() {
+ return {
+ init: vi.fn(async () => undefined),
+ disconnect: vi.fn(async () => undefined),
+ getTools: vi.fn(() => ({
+ [computerUseToolName]: { execute: vi.fn(async () => ({})) },
+ })),
+ getServerStatuses: vi.fn(() => [
+ {
+ name: "builtin-computer-use",
+ connected: true,
+ toolCount: 1,
+ toolNames: [computerUseToolName],
+ transport: "stdio" as const,
+ },
+ ]),
+ };
+}
function makeProviderSession(
threadId: ThreadId,
@@ -259,12 +291,15 @@ function makeLayer(
makeMcpManager?: NonNullable<AgentControllerLiveOptions["makeMcpManager"]>,
baseDir?: string,
usageLedger: BotUsageLedgerShape = makeUsageLedger().service,
- memory?: Pick<AgentControllerLiveOptions, "entityMemoryRepository" | "memoryCandidateRepository">,
+ overrides?: Pick<
+ AgentControllerLiveOptions,
+ "resolveComputerUseServer" | "entityMemoryRepository" | "memoryCandidateRepository"
+ >,
) {
return makeAgentControllerLive({
makeMastraHarness: factory,
...(makeMcpManager ? { makeMcpManager } : {}),
- ...memory,
+ ...overrides,
makeBotBrowser: () =>
({
tools: {},
@@ -295,10 +330,13 @@ function provideController<A, E>(
makeMcpManager?: NonNullable<AgentControllerLiveOptions["makeMcpManager"]>,
baseDir?: string,
usageLedger?: BotUsageLedgerShape,
- memory?: Pick<AgentControllerLiveOptions, "entityMemoryRepository" | "memoryCandidateRepository">,
+ overrides?: Pick<
+ AgentControllerLiveOptions,
+ "resolveComputerUseServer" | "entityMemoryRepository" | "memoryCandidateRepository"
+ >,
) {
return effect.pipe(
- Effect.provide(makeLayer(bridge, factory, makeMcpManager, baseDir, usageLedger, memory)),
+ Effect.provide(makeLayer(bridge, factory, makeMcpManager, baseDir, usageLedger, overrides)),
Effect.orDie,
);
}
@@ -1672,6 +1710,189 @@ describe("AgentControllerLive", () => {
);
});
+ it.effect("keeps Computer Use approval data and desktop content out of runtime events", () => {
+ const bridge = makeBridge();
+ const mastra = makeMastraHarness();
+ const toolName = computerUseToolName;
+ const mcpManager = computerUseMcpManager();
+ const server = computerUseServer();
+
+ return provideController(
+ Effect.gen(function* () {
+ const controller = yield* AgentController;
+ yield* resolveCodex(controller);
+ yield* controller.startSession(codexThreadId, {
+ threadId: codexThreadId,
+ provider: ProviderDriverKind.make("codex"),
+ providerInstanceId: codexInstanceId,
+ modelSelection: codexSelection,
+ runtimeMode: "full-access",
+ mcpServers: [server],
+ });
+
+ const events: ProviderRuntimeEvent[] = [];
+ const eventsFiber = yield* controller.streamEvents.pipe(
+ Stream.runForEach((event) => Effect.sync(() => events.push(event))),
+ Effect.forkChild({ startImmediately: true }),
+ );
+ yield* Effect.yieldNow;
+ yield* controller.sendTurn({ threadId: codexThreadId, input: "Control this Mac." });
+ mastra.emit({
+ type: "tool_start",
+ toolCallId: "computer-use-1",
+ toolName,
+ args: { text: "typed secret", app: "Private App", path: "/private/tester" },
+ } as AgentControllerEvent);
+ mastra.emit({
+ type: "tool_update",
+ toolCallId: "computer-use-1",
+ partialResult: { windowTitle: "Private Window" },
+ } as AgentControllerEvent);
+ mastra.emit({
+ type: "tool_approval_required",
+ toolCallId: "computer-use-1",
+ toolName,
+ args: { text: "typed secret", app: "Private App" },
+ } as AgentControllerEvent);
+ yield* controller.respondToRequest({
+ threadId: codexThreadId,
+ requestId: ApprovalRequestId.make("computer-use-1"),
+ decision: "acceptForSession",
+ });
+ mastra.emit({
+ type: "tool_end",
+ toolCallId: "computer-use-1",
+ result: { screenshot: "raw frame", applicationContent: "private content" },
+ } as AgentControllerEvent);
+ mastra.emit({
+ type: "tool_approval_required",
+ toolCallId: "computer-use-2",
+ toolName,
+ args: { text: "deny this" },
+ } as AgentControllerEvent);
+ yield* controller.respondToRequest({
+ threadId: codexThreadId,
+ requestId: ApprovalRequestId.make("computer-use-2"),
+ decision: "decline",
+ });
+ mastra.emit({
+ type: "error",
+ error: new Error("Private Window at /private/tester failed"),
+ } as AgentControllerEvent);
+ yield* Effect.yieldNow;
+ yield* Fiber.interrupt(eventsFiber);
+
+ const serialized = JSON.stringify(events);
+ expect(serialized).not.toContain("typed secret");
+ expect(serialized).not.toContain("Private App");
+ expect(serialized).not.toContain("Private Window");
+ expect(serialized).not.toContain("private/tester");
+ expect(serialized).not.toContain("raw frame");
+ expect(serialized).not.toContain("private content");
+ const approval = events.find(
+ (event): event is Extract<ProviderRuntimeEvent, { readonly type: "request.opened" }> =>
+ event.type === "request.opened" && event.requestId === "computer-use-1",
+ );
+ assert.isDefined(approval);
+ assert.isDefined(approval.payload.options);
+ expect(approval.payload.options.map((option) => option.decision)).toEqual([
+ "accept",
+ "decline",
+ ]);
+ expect(mastra.session.respondToToolApproval).toHaveBeenCalledWith({
+ toolCallId: "computer-use-1",
+ decision: "approve",
+ });
+ expect(mastra.session.respondToToolApproval).toHaveBeenCalledWith({
+ toolCallId: "computer-use-2",
+ decision: "decline",
+ });
+ expect(mastra.session.permissions.setForTool).not.toHaveBeenCalledWith({
+ toolName,
+ policy: "allow",
+ });
+ mastra.finishSend();
+ }),
+ bridge.service,
+ mastra.factory,
+ (() => mcpManager as never) as NonNullable<AgentControllerLiveOptions["makeMcpManager"]>,
+ undefined,
+ undefined,
+ {
+ resolveComputerUseServer: async () => ({
+ command: "/local/launcher",
+ args: ["mcp"],
+ env: {},
+ }),
+ },
+ );
+ });
+
+ it.effect("releases Computer Use when session setup and deletion fail", () => {
+ const bridge = makeBridge();
+ const mastra = makeMastraHarness();
+ const mcpManager = computerUseMcpManager();
+ const nextThreadId = ThreadId.make("thread-mastra-codex-next");
+ vi.mocked(mastra.session.state.set).mockRejectedValueOnce(new Error("setup failed"));
+
+ return provideController(
+ Effect.gen(function* () {
+ const controller = yield* AgentController;
+ yield* resolveCodex(controller);
+ yield* controller
+ .startSession(codexThreadId, {
+ threadId: codexThreadId,
+ provider: ProviderDriverKind.make("codex"),
+ providerInstanceId: codexInstanceId,
+ modelSelection: codexSelection,
+ runtimeMode: "full-access",
+ mcpServers: [computerUseServer()],
+ })
+ .pipe(Effect.ignore);
+
+ yield* controller.resolveEngine({
+ threadId: nextThreadId,
+ engine: { provider: "codex", model: "gpt-5.6-sol" },
+ fallback: codexSelection,
+ mode: "default",
+ botConversation: true,
+ });
+ yield* controller.startSession(nextThreadId, {
+ threadId: nextThreadId,
+ provider: ProviderDriverKind.make("codex"),
+ providerInstanceId: codexInstanceId,
+ modelSelection: codexSelection,
+ runtimeMode: "full-access",
+ mcpServers: [computerUseServer()],
+ });
+
+ mastra.deleteSession.mockRejectedValueOnce(new Error("delete failed"));
+ yield* controller.stopSession({ threadId: nextThreadId }).pipe(Effect.ignore);
+ yield* controller.startSession(codexThreadId, {
+ threadId: codexThreadId,
+ provider: ProviderDriverKind.make("codex"),
+ providerInstanceId: codexInstanceId,
+ modelSelection: codexSelection,
+ runtimeMode: "full-access",
+ mcpServers: [computerUseServer()],
+ });
+ expect(mcpManager.init).toHaveBeenCalledTimes(3);
+ }),
+ bridge.service,
+ mastra.factory,
+ (() => mcpManager as never) as NonNullable<AgentControllerLiveOptions["makeMcpManager"]>,
+ undefined,
+ undefined,
+ {
+ resolveComputerUseServer: async () => ({
+ command: "/local/launcher",
+ args: ["mcp"],
+ env: {},
+ }),
+ },
+ );
+ });
+
it.effect("creates a remote Mastra workspace for the bot sandbox provider", () => {
const bridge = makeBridge();
const mastra = makeMastraHarness();
diff --git a/apps/server/src/provider/Layers/AgentController.ts b/apps/server/src/provider/Layers/AgentController.ts
index b5538fbb1..8f77d003b 100644
--- a/apps/server/src/provider/Layers/AgentController.ts
+++ b/apps/server/src/provider/Layers/AgentController.ts
@@ -32,6 +32,7 @@ import {
AKERU_PRODUCT_FEEDBACK_TOOL_NAME,
type AkeruMemoryThreadAccess,
} from "@t3tools/contracts";
+import { HostProcessPlatform } from "@t3tools/shared/hostProcess";
import * as Effect from "effect/Effect";
import * as Layer from "effect/Layer";
import * as PubSub from "effect/PubSub";
@@ -83,6 +84,11 @@ import {
} from "../AkeruToolRuntime.ts";
import type { BotBrowser, BotBrowserAttachment, CreateBotBrowserInput } from "../botBrowser.ts";
import { AkeruSessionResources } from "../AkeruSessionResources.ts";
+import {
+ isCodexComputerUseServer,
+ isCodexComputerUseTool,
+ resolveCodexComputerUseServer,
+} from "../CodexComputerUse.ts";
import type { AkeruBotWorkspace, CreateRemoteBotWorkspaceInput } from "../botWorkspace.ts";
import {
botRuntimeResourceScope,
@@ -151,6 +157,7 @@ export interface AgentControllerLiveOptions {
input: CreateRemoteBotWorkspaceInput,
) => Promise<AkeruBotWorkspace | Workspace>;
readonly makeBotBrowser?: (input: CreateBotBrowserInput) => BotBrowser;
+ readonly resolveComputerUseServer?: typeof resolveCodexComputerUseServer;
readonly entityMemoryRepository?: EntityMemoryRepositoryShape;
readonly memoryCandidateRepository?: MemoryCandidateRepositoryShape;
}
@@ -163,6 +170,12 @@ function failureDetail(cause: unknown): string {
return cause instanceof Error ? cause.message : String(cause);
}
+function sessionFailureDetail(active: Pick<ActiveSession, "mcpServerIds">, cause: unknown): string {
+ return active.mcpServerIds.some((id) => isCodexComputerUseServer(String(id)))
+ ? "Computer Use session failed."
+ : failureDetail(cause);
+}
+
function nowIso(): string {
return new Date().toISOString();
}
@@ -319,6 +332,7 @@ function itemType(
const make = (options?: AgentControllerLiveOptions) =>
Effect.gen(function* () {
const config = yield* ServerConfig;
+ const hostPlatform = yield* HostProcessPlatform;
const legacyProviderBridge = yield* LegacyProviderBridge;
const botUsageLedger = yield* BotUsageLedger;
const runtimeContext = yield* Effect.context<never>();
@@ -362,12 +376,16 @@ const make = (options?: AgentControllerLiveOptions) =>
const subscriptionAuth = SubscriptionAuthService.forSecretsDir(config.secretsDir);
const sessionResources = new AkeruSessionResources({
stateDir: config.stateDir,
+ hostPlatform,
toMcpServerConfigs,
onMcpServerConnectionFailure: (serverId) =>
subscriptionAuth.recordMcpRequestFailure(serverId, "The MCP server failed to connect."),
...(options?.makeMcpManager ? { makeMcpManager: options.makeMcpManager } : {}),
...(options?.makeRemoteWorkspace ? { makeRemoteWorkspace: options.makeRemoteWorkspace } : {}),
...(options?.makeBotBrowser ? { makeBotBrowser: options.makeBotBrowser } : {}),
+ ...(options?.resolveComputerUseServer
+ ? { resolveComputerUseServer: options.resolveComputerUseServer }
+ : {}),
});
const botInbox = BotInboxService.forSecretsDir(config.secretsDir);
const toolRuntime = createAkeruToolRuntime({
@@ -698,8 +716,10 @@ const make = (options?: AgentControllerLiveOptions) =>
payload: {
itemType: itemType(event.toolName),
status: "inProgress",
- title: event.toolName,
- data: { args: event.args },
+ title: isCodexComputerUseTool(event.toolName) ? "Computer Use" : event.toolName,
+ data: isCodexComputerUseTool(event.toolName)
+ ? { action: "computer-use" }
+ : { args: event.args },
},
});
return;
@@ -713,7 +733,9 @@ const make = (options?: AgentControllerLiveOptions) =>
payload: {
itemType: itemType(active.toolNames.get(event.toolCallId) ?? "tool"),
status: "inProgress",
- data: { partialResult: event.partialResult },
+ data: isCodexComputerUseTool(active.toolNames.get(event.toolCallId) ?? "")
+ ? { action: "computer-use" }
+ : { partialResult: event.partialResult },
},
});
return;
@@ -744,8 +766,10 @@ const make = (options?: AgentControllerLiveOptions) =>
payload: {
itemType: itemType(toolName),
status: event.isError ? "failed" : event.denied ? "declined" : "completed",
- title: toolName,
- data: { result: event.result },
+ title: isCodexComputerUseTool(toolName) ? "Computer Use" : toolName,
+ data: isCodexComputerUseTool(toolName)
+ ? { action: "computer-use" }
+ : { result: event.result },
},
});
return;
@@ -766,14 +790,19 @@ const make = (options?: AgentControllerLiveOptions) =>
type: "request.opened",
payload: {
requestType: "dynamic_tool_call",
- detail:
- event.toolName === AKERU_PRODUCT_FEEDBACK_TOOL_NAME
+ detail: isCodexComputerUseTool(event.toolName)
+ ? "Allow Computer Use?"
+ : event.toolName === AKERU_PRODUCT_FEEDBACK_TOOL_NAME
? "Review product feedback"
: `Allow ${event.toolName}?`,
- toolName: event.toolName,
- args: event.args,
- options:
- event.toolName === AKERU_PRODUCT_FEEDBACK_TOOL_NAME
+ toolName: isCodexComputerUseTool(event.toolName) ? "Computer Use" : event.toolName,
+ args: isCodexComputerUseTool(event.toolName) ? undefined : event.args,
+ options: isCodexComputerUseTool(event.toolName)
+ ? [
+ { decision: "accept", label: "Allow" },
+ { decision: "decline", label: "Decline" },
+ ]
+ : event.toolName === AKERU_PRODUCT_FEEDBACK_TOOL_NAME
? [
{ decision: "accept", label: "Add to feedback draft" },
{ decision: "decline", label: "Cancel" },
@@ -834,17 +863,19 @@ const make = (options?: AgentControllerLiveOptions) =>
},
});
return;
- case "error":
+ case "error": {
+ const detail = sessionFailureDetail(active, event.error);
publish({
...baseEvent(threadId, active, turn?.turnId),
type: "runtime.error",
payload: {
- message: event.error.message || "Mastra agent failed.",
+ message: detail,
class: "provider_error",
},
});
- finishTurn(threadId, active, "failed", event.error.message || "Mastra agent failed.");
+ finishTurn(threadId, active, "failed", detail);
return;
+ }
case "agent_end":
if (event.reason === "suspended") {
if (turn) turn.waiting = true;
@@ -942,12 +973,30 @@ const make = (options?: AgentControllerLiveOptions) =>
const workspaceId = botWorkspaceIdentity(workspaceResourceKey);
const existing = sessions.get(key);
const resolved = resolvedByThread.get(key);
+ const mcpServers = input.mcpServers ?? [];
+ if (!resolved) {
+ return yield* new AgentControllerRuntimeError({
+ operation: "startSession",
+ detail: `Thread '${threadId}' has no resolved engine.`,
+ });
+ }
+ if (
+ mcpServers.some((server) => isCodexComputerUseServer(String(server.id))) &&
+ resolved.provider !== ProviderDriverKind.make("codex")
+ ) {
+ return yield* new AgentControllerRuntimeError({
+ operation: "startSession",
+ detail: "Computer Use requires a Codex bot.",
+ });
+ }
if (
existing?.workspaceResourceKey === workspaceResourceKey &&
existing.cwd === input.cwd &&
resolved &&
existing.provider === resolved.provider &&
- existing.providerInstanceId === resolved.providerInstanceId
+ existing.providerInstanceId === resolved.providerInstanceId &&
+ existing.mcpServerIds.length === (input.mcpServers?.length ?? 0) &&
+ existing.mcpServerIds.every((id) => input.mcpServers?.some((server) => server.id === id))
) {
existing.runtimeMode = input.runtimeMode;
const toolSession = { ...existing.toolSession };
@@ -991,13 +1040,6 @@ const make = (options?: AgentControllerLiveOptions) =>
previousWorkspaceResourceKey !== workspaceResourceKey,
);
}
- if (!resolved) {
- return yield* new AgentControllerRuntimeError({
- operation: "startSession",
- detail: `Thread '${threadId}' has no resolved engine.`,
- });
- }
- const mcpServers = input.mcpServers ?? [];
const resources = yield* runMastra("resources.acquire", () =>
sessionResources.acquire({
threadId: key,
@@ -1112,67 +1154,77 @@ const make = (options?: AgentControllerLiveOptions) =>
),
),
);
- yield* runMastra("state.set", () =>
- session.state.set({
- ...(input.cwd ? { projectPath: input.cwd } : {}),
- yolo: false,
- botConversation: resolved.botConversation,
- }),
- );
- yield* runMastra("model.switch", () =>
- session.model.switch({ modelId: resolved.mastraModelId }),
- );
- const modeId = mastraModeId(resolved.mode);
- if (session.mode.get() !== modeId) {
- yield* runMastra("mode.switch", () => session.mode.switch({ modeId }));
- }
- yield* Effect.forEach(
- ["read", "edit", "execute", "mcp", "other"] as const,
- (category) =>
- runMastra("permissions.setForCategory", () =>
- session.permissions.setForCategory({
- category,
- policy: permissionPolicy(input.runtimeMode, category),
- }),
- ),
- { discard: true },
- );
- yield* Effect.forEach(
- new Set([
- ...toolRuntime.toolsForThread(key).map((tool) => tool.id),
- ...Object.keys(sessionResources.getConnectorTools(key)),
- AKERU_PRODUCT_FEEDBACK_TOOL_NAME,
- "RestartMcpServers",
- ]),
- (toolName) =>
- runMastra("permissions.setForTool", () =>
- session.permissions.setForTool({ toolName, policy: "ask" }),
- ),
- { discard: true },
- );
- const createdAt = nowIso();
- const activeWithoutUnsubscribe = {
- session,
- provider: resolved.provider,
- providerInstanceId: resolved.providerInstanceId,
- cwd: input.cwd,
- createdAt,
- mcpServerIds: mcpServers.map((server) => server.id),
- runtimeMode: input.runtimeMode,
- model: resolved.modelSelection.model,
- status: "ready" as const,
- activeTurn: null,
- toolNames: new Map<string, string>(),
- approvalRequests: new Map(),
- connectorSessionApprovals: new Set<string>(),
- toolSession,
- workspaceResourceKey,
- };
- const unsubscribe = session.subscribe((event) => {
- const active = sessions.get(key);
- if (active) handleControllerEvent(threadId, active, event);
+ const cleanupCreatedSession = Effect.gen(function* () {
+ yield* runMastra("deleteSession", () =>
+ bundle.controller.deleteSession({ resourceId: key }),
+ ).pipe(Effect.ignoreCause({ log: true }));
+ yield* runMastra("resources.release", () =>
+ sessionResources.release(key, { destroy: true }),
+ ).pipe(Effect.ignoreCause({ log: true }));
+ toolRuntime.unregisterSession(key);
});
- const active: ActiveSession = { ...activeWithoutUnsubscribe, unsubscribe };
+ const active = yield* Effect.gen(function* () {
+ yield* runMastra("state.set", () =>
+ session.state.set({
+ ...(input.cwd ? { projectPath: input.cwd } : {}),
+ yolo: false,
+ botConversation: resolved.botConversation,
+ }),
+ );
+ yield* runMastra("model.switch", () =>
+ session.model.switch({ modelId: resolved.mastraModelId }),
+ );
+ const modeId = mastraModeId(resolved.mode);
+ if (session.mode.get() !== modeId) {
+ yield* runMastra("mode.switch", () => session.mode.switch({ modeId }));
+ }
+ yield* Effect.forEach(
+ ["read", "edit", "execute", "mcp", "other"] as const,
+ (category) =>
+ runMastra("permissions.setForCategory", () =>
+ session.permissions.setForCategory({
+ category,
+ policy: permissionPolicy(input.runtimeMode, category),
+ }),
+ ),
+ { discard: true },
+ );
+ yield* Effect.forEach(
+ new Set([
+ ...toolRuntime.toolsForThread(key).map((tool) => tool.id),
+ ...Object.keys(sessionResources.getConnectorTools(key)),
+ AKERU_PRODUCT_FEEDBACK_TOOL_NAME,
+ "RestartMcpServers",
+ ]),
+ (toolName) =>
+ runMastra("permissions.setForTool", () =>
+ session.permissions.setForTool({ toolName, policy: "ask" }),
+ ),
+ { discard: true },
+ );
+ const unsubscribe = session.subscribe((event) => {
+ const current = sessions.get(key);
+ if (current) handleControllerEvent(threadId, current, event);
+ });
+ return {
+ session,
+ provider: resolved.provider,
+ providerInstanceId: resolved.providerInstanceId,
+ cwd: input.cwd,
+ createdAt: nowIso(),
+ mcpServerIds: mcpServers.map((server) => server.id),
+ runtimeMode: input.runtimeMode,
+ model: resolved.modelSelection.model,
+ status: "ready" as const,
+ activeTurn: null,
+ toolNames: new Map<string, string>(),
+ approvalRequests: new Map(),
+ connectorSessionApprovals: new Set<string>(),
+ toolSession,
+ workspaceResourceKey,
+ unsubscribe,
+ } satisfies ActiveSession;
+ }).pipe(Effect.onError(() => cleanupCreatedSession));
sessions.set(key, active);
publish({
...baseEvent(threadId, active),
@@ -1281,7 +1333,7 @@ const make = (options?: AgentControllerLiveOptions) =>
}
})
.catch((cause: unknown) => {
- const detail = failureDetail(cause);
+ const detail = sessionFailureDetail(active, cause);
publish({
...baseEvent(input.threadId, active, turnId),
type: "runtime.error",
@@ -1342,6 +1394,7 @@ const make = (options?: AgentControllerLiveOptions) =>
if (
input.decision === "acceptForSession" &&
!runtimeToolId &&
+ !isCodexComputerUseTool(toolName) &&
!akeruActionNeedsApproval(toolName, toolInput) &&
toolName !== AKERU_PRODUCT_FEEDBACK_TOOL_NAME
) {
@@ -1403,16 +1456,25 @@ const make = (options?: AgentControllerLiveOptions) =>
if (!active) {
const legacySessions = yield* legacyProviderBridge.listSessions();
if (legacySessions.some((session) => session.threadId === input.threadId)) {
- yield* legacyProviderBridge.stopSession(input);
- yield* runMastra("resources.release", () =>
- sessionResources.release(key, { destroy: destroyResources }),
- ).pipe(Effect.ignoreCause({ log: true }));
- legacyResourceIdentity.delete(key);
+ yield* legacyProviderBridge.stopSession(input).pipe(
+ Effect.ensuring(
+ Effect.gen(function* () {
+ yield* runMastra("resources.release", () =>
+ sessionResources.release(key, { destroy: destroyResources }),
+ ).pipe(Effect.ignoreCause({ log: true }));
+ legacyResourceIdentity.delete(key);
+ }),
+ ),
+ );
return;
}
if (
usesMastraCode(resolvedByThread.get(key)?.provider ?? ProviderDriverKind.make("codex"))
) {
+ yield* runMastra("resources.release", () =>
+ sessionResources.release(key, { destroy: destroyResources }),
+ ).pipe(Effect.ignoreCause({ log: true }));
+ toolRuntime.unregisterSession(key);
return;
}
return yield* legacyProviderBridge.stopSession(input);
@@ -1421,13 +1483,20 @@ const make = (options?: AgentControllerLiveOptions) =>
active.session.abort();
active.unsubscribe();
publishSessionState(input.threadId, active, "stopped");
- yield* runMastra("deleteSession", () => bundle.controller.deleteSession({ resourceId: key }));
- yield* runMastra("resources.release", () =>
- sessionResources.release(key, { destroy: destroyResources }),
- ).pipe(Effect.ignoreCause({ log: true }));
- toolRuntime.unregisterSession(key);
- sessions.delete(key);
- memoryUsageByThread.delete(key);
+ yield* runMastra("deleteSession", () =>
+ bundle.controller.deleteSession({ resourceId: key }),
+ ).pipe(
+ Effect.ensuring(
+ Effect.gen(function* () {
+ yield* runMastra("resources.release", () =>
+ sessionResources.release(key, { destroy: destroyResources }),
+ ).pipe(Effect.ignoreCause({ log: true }));
+ toolRuntime.unregisterSession(key);
+ sessions.delete(key);
+ memoryUsageByThread.delete(key);
+ }),
+ ),
+ );
});
const stopSession: AgentControllerShape["stopSession"] = (input) =>
diff --git a/apps/web/src/components/plugins/pluginPresentation.test.ts b/apps/web/src/components/plugins/pluginPresentation.test.ts
index 121cefb21..8cafa1fab 100644
--- a/apps/web/src/components/plugins/pluginPresentation.test.ts
+++ b/apps/web/src/components/plugins/pluginPresentation.test.ts
@@ -92,7 +92,7 @@ describe("plugin presentation", () => {
buildPluginSections({ plugins: catalog, query: "", filter: "All" }).flatMap((section) =>
section.plugins.map((plugin) => plugin.id),
),
- ).toHaveLength(51);
+ ).toHaveLength(52);
expect(
buildPluginSections({
plugins: catalog,
diff --git a/apps/web/src/components/sidebar/SidebarChrome.test.ts b/apps/web/src/components/sidebar/SidebarChrome.test.ts
index e7b682602..dbdcad7ef 100644
--- a/apps/web/src/components/sidebar/SidebarChrome.test.ts
+++ b/apps/web/src/components/sidebar/SidebarChrome.test.ts
@@ -1,11 +1,15 @@
// @effect-diagnostics nodeBuiltinImport:off - This integration guard reads related source files.
import * as NodeFS from "node:fs";
-import { McpServerId, type McpServer } from "@t3tools/contracts";
+import { BotId, McpServerId, ThreadId, type McpServer } from "@t3tools/contracts";
import { describe, expect, it } from "vite-plus/test";
import { loadCatalog } from "../../../../../plugins";
-import { formatEnabledPluginStatus, summarizeEnabledPlugins } from "./SidebarChrome";
+import {
+ findActiveComputerUseControl,
+ formatEnabledPluginStatus,
+ summarizeEnabledPlugins,
+} from "./SidebarChrome";
const catalogPlugin = loadCatalog().find((plugin) => plugin.id === "exa");
if (!catalogPlugin) throw new TypeError("Exa must remain in the plugin catalog.");
@@ -39,6 +43,95 @@ describe("sidebar footer", () => {
expect(summary.enabledPlugins).toEqual([catalogPlugin]);
});
+ it("shows only the bot that holds an enabled Computer Use session", () => {
+ const botId = BotId.make("bot-1");
+ const input: Parameters<typeof findActiveComputerUseControl>[0] = {
+ mcpServers: [server("builtin-computer-use")],
+ bots: [
+ {
+ id: botId,
+ name: "Operator",
+ title: "Operator",
+ label: null,
+ description: "",
+ engine: null,
+ sandbox: null,
+ runtimeMode: "approval-required",
+ usageCap: null,
+ voiceEnabled: false,
+ groupId: null,
+ disabledMcpServerIds: [],
+ avatar: { kind: "blob", shape: "circle", color: "#5B7FD4" },
+ archivedAt: null,
+ createdAt: "2026-08-27T00:00:00.000Z",
+ updatedAt: "2026-08-27T00:00:00.000Z",
+ },
+ ],
+ threads: [
+ {
+ id: ThreadId.make("thread-1"),
+ botId,
+ projectId: "project-1" as never,
+ title: "Control",
+ modelSelection: { instanceId: "codex" as never, model: "gpt-5.6-sol" },
+ runtimeMode: "approval-required",
+ interactionMode: "default",
+ branch: null,
+ worktreePath: null,
+ latestTurn: null,
+ createdAt: "2026-08-27T00:00:00.000Z",
+ updatedAt: "2026-08-27T00:00:00.000Z",
+ archivedAt: null,
+ settledOverride: null,
+ settledAt: null,
+ session: {
+ threadId: ThreadId.make("thread-1"),
+ status: "ready",
+ providerName: "codex",
+ runtimeMode: "approval-required",
+ mcpServerIds: [McpServerId.make("builtin-computer-use")],
+ activeTurnId: null,
+ lastError: null,
+ updatedAt: "2026-08-27T00:00:00.000Z",
+ },
+ latestUserMessageAt: null,
+ hasPendingApprovals: false,
+ hasPendingUserInput: false,
+ hasActionableProposedPlan: false,
+ },
+ ],
+ };
+ const control = findActiveComputerUseControl(input);
+
+ expect(control).toEqual({ threadId: ThreadId.make("thread-1"), botName: "Operator" });
+ for (const status of ["starting", "error", "stopped"] as const) {
+ expect(
+ findActiveComputerUseControl({
+ ...input,
+ threads: input.threads.map((thread) => ({
+ ...thread,
+ session: thread.session ? { ...thread.session, status } : null,
+ })),
+ }),
+ ).toBeNull();
+ }
+ expect(
+ findActiveComputerUseControl({
+ ...input,
+ mcpServers: [server("builtin-computer-use", false)],
+ }),
+ ).toBeNull();
+ expect(
+ findActiveComputerUseControl({
+ ...input,
+ threads: input.threads.map((thread) => ({
+ ...thread,
+ session: thread.session ? { ...thread.session, mcpServerIds: [] } : null,
+ })),
+ }),
+ ).toBeNull();
+ });
+
it("keeps the error inbox in Settings instead of the sidebar", () => {
const source = NodeFS.readFileSync(new URL("./SidebarChrome.tsx", import.meta.url), "utf8");
diff --git a/apps/web/src/components/sidebar/SidebarChrome.tsx b/apps/web/src/components/sidebar/SidebarChrome.tsx
index 3eb88668b..bc3b404dd 100644
--- a/apps/web/src/components/sidebar/SidebarChrome.tsx
+++ b/apps/web/src/components/sidebar/SidebarChrome.tsx
@@ -5,10 +5,16 @@ import {
PlugSocketIcon,
Settings02Icon,
} from "@hugeicons/core-free-icons";
-import type { EnvironmentId, McpServer } from "@t3tools/contracts";
+import type {
+ EnvironmentId,
+ McpServer,
+ OrchestrationBot,
+ OrchestrationThreadShell,
+ ThreadId,
+} from "@t3tools/contracts";
import { Link } from "@tanstack/react-router";
import type { ReactNode } from "react";
-import { memo, useCallback } from "react";
+import { memo, useCallback, useState } from "react";
import {
loadCatalog,
resolveCatalogInstallations,
@@ -21,7 +27,10 @@ import { openPlugins } from "../../pluginsDialogStore";
import { openUsage } from "../../usageDialogStore";
import { openProductFeedback } from "../../productFeedbackStore";
import { usePrimaryEnvironmentId } from "../../state/environments";
-import { environmentMcpServersAtom } from "../../state/mcpServers";
+import { environmentMcpServersAtom, mcpServerEnvironment } from "../../state/mcpServers";
+import { environmentSnapshotAtom } from "../../state/shell";
+import { threadEnvironment } from "../../state/threads";
+import { useAtomCommand } from "../../state/use-atom-command";
import { AkeruMark } from "../AkeruMark";
import { T3ConnectSidebarAvatar, T3ConnectSidebarSignIn } from "../clerk/T3ConnectSidebarSignIn";
import { PluginLogoImage } from "../plugins/PluginsCatalog";
@@ -35,6 +44,7 @@ import {
useEnvironmentStageLabel,
} from "../SidebarStageBackdrop";
import { Badge } from "../ui/badge";
+import { Button } from "../ui/button";
import { AppIcon } from "../ui/app-icon";
import {
SidebarFooter,
@@ -120,6 +130,38 @@ function SidebarBrand({ onBackdrop }: { onBackdrop: boolean }) {
}
const PLUGIN_CATALOG = loadCatalog();
+const COMPUTER_USE_SERVER_ID = "builtin-computer-use";
+
+export interface ActiveComputerUseControl {
+ readonly threadId: ThreadId;
+ readonly botName: string;
+}
+
+export function findActiveComputerUseControl(input: {
+ readonly threads: readonly OrchestrationThreadShell[];
+ readonly bots: readonly OrchestrationBot[];
+ readonly mcpServers: readonly McpServer[];
+}): ActiveComputerUseControl | null {
+ const server = input.mcpServers.find(
+ (candidate) => candidate.id === COMPUTER_USE_SERVER_ID && candidate.enabled,
+ );
+ if (!server) return null;
+ for (const thread of input.threads) {
+ if (
+ !thread.session ||
+ thread.session.providerName?.toLowerCase() !== "codex" ||
+ !thread.session.mcpServerIds?.includes(server.id) ||
+ (thread.session.status !== "ready" && thread.session.status !== "running")
+ ) {
+ continue;
+ }
+ const botId = thread.respondingBotId ?? thread.botId;
+ const bot = input.bots.find((candidate) => candidate.id === botId);
+ if (!bot || bot.disabledMcpServerIds.includes(server.id)) continue;
+ return { threadId: thread.id, botName: bot.name };
+ }
+ return null;
+}
export function formatEnabledPluginStatus(enabledCount: number): string {
if (enabledCount === 0) return "No plugins enabled";
@@ -224,6 +266,81 @@ function SidebarPluginSummary({ onClick }: { readonly onClick: () => void }) {
return <SidebarPluginSummaryForEnvironment environmentId={environmentId} onClick={onClick} />;
}
+function ComputerUseControlForEnvironment({
+ environmentId,
+}: {
+ readonly environmentId: EnvironmentId;
+}) {
+ const snapshot = useAtomValue(environmentSnapshotAtom(environmentId));
+ const stopSession = useAtomCommand(threadEnvironment.stopSession);
+ const disableServer = useAtomCommand(mcpServerEnvironment.disable);
+ const [pending, setPending] = useState(false);
+ const control = snapshot
+ ? findActiveComputerUseControl({
+ threads: snapshot.threads,
+ bots: snapshot.bots,
+ mcpServers: snapshot.mcpServers ?? [],
+ })
+ : null;
+ if (!control) return null;
+
+ const stop = async () => {
+ setPending(true);
+ try {
+ await stopSession({ environmentId, input: { threadId: control.threadId } });
+ } finally {
+ setPending(false);
+ }
+ };
+ const revoke = async () => {
+ setPending(true);
+ try {
+ const stopped = await stopSession({
+ environmentId,
+ input: { threadId: control.threadId },
+ });
+ if (stopped._tag === "Success") {
+ await disableServer({
+ environmentId,
+ input: { mcpServerId: COMPUTER_USE_SERVER_ID as McpServer["id"] },
+ });
+ }
+ } finally {
+ setPending(false);
+ }
+ };
+
+ return (
+ <div className="rounded-xl border border-amber-500/30 bg-amber-500/10 p-2.5" role="status">
+ <div className="flex items-center gap-2 text-xs font-medium">
+ <span className="size-2 rounded-full bg-amber-500" aria-hidden="true" />
+ <span className="min-w-0 truncate">{control.botName} controls this Mac</span>
+ </div>
+ <div className="mt-2 flex gap-2">
+ <Button className="h-7 flex-1 text-xs" disabled={pending} size="sm" onClick={stop}>
+ Stop
+ </Button>
+ <Button
+ aria-label="Revoke Computer Use for all bots"
+ className="h-7 flex-1 text-xs"
+ disabled={pending}
+ size="sm"
+ title="Disable Computer Use for all bots"
+ variant="destructive-outline"
+ onClick={revoke}
+ >
+ Revoke
+ </Button>
+ </div>
+ </div>
+ );
+}
+
+function ComputerUseControl() {
+ const environmentId = usePrimaryEnvironmentId();
+ return environmentId ? <ComputerUseControlForEnvironment environmentId={environmentId} /> : null;
+}
+
function SidebarUtilityItem({
icon,
label,
@@ -279,6 +396,7 @@ export const SidebarChromeFooter = memo(function SidebarChromeFooter() {
return (
<SidebarFooter className="max-h-[min(45dvh,22rem)] shrink-0 overflow-y-auto overscroll-contain p-[var(--sidebar-content-inset)]">
<div className="flex flex-col gap-2 empty:hidden group-data-[collapsible=icon]:hidden">
+ <ComputerUseControl />
<SidebarProviderUpdatePill />
<SidebarUpdateArchitectureWarning />
</div>
diff --git a/docs/internals/plugin-lifecycle-verification.md b/docs/internals/plugin-lifecycle-verification.md
index 9a8e0207e..e80d280bf 100644
--- a/docs/internals/plugin-lifecycle-verification.md
+++ b/docs/internals/plugin-lifecycle-verification.md
@@ -2,7 +2,7 @@
Milestone 13 keeps catalog discovery separate from connection verification. An entry stays non-installable until its real install, authentication, safe read, approved write, disable, reconnect, and removal lifecycle passes. `verification-pending` means that local lifecycle proof is missing. `approval-pending` means an external vendor, administrator, allowlist, or first-party connector is still required.
-`plugins/lifecycle-matrix.test.ts` is the catalog gate. It fixes the directory at 51 IDs, keeps four verified `builtin-<id>` recipes installable, and keeps the other 47 entries pending. It also checks Featured order, consequential approval coverage, broker identity, Custom MCP independence, legacy built-in display, and the Executor, Typefully, Paper, and PayPal blockers.
+`plugins/lifecycle-matrix.test.ts` is the catalog gate. It fixes the directory at 52 IDs, keeps four verified `builtin-<id>` recipes installable, and keeps the other 48 entries pending. It also checks Featured order, consequential approval coverage, broker identity, Custom MCP independence, legacy built-in display, and the Computer Use, Executor, Typefully, Paper, and PayPal blockers.
## Runtime proof
diff --git a/docs/user/plugins.md b/docs/user/plugins.md
index d5b429c88..33899fb12 100644
--- a/docs/user/plugins.md
+++ b/docs/user/plugins.md
@@ -15,6 +15,8 @@ The available action matches the current state:
Approval-pending plugins remain visible, but Akeru disables **Connect** and shows the vendor blocker. Plugins removed from the directory remain under **Installed** until you remove them.
+Codex Computer Use runs only on the local Mac. It does not use a hosted desktop or remote-control service. When a Codex bot starts control, macOS requests Screen Recording and Accessibility access if the helper does not have them. Akeru allows one controlling bot at a time and shows its name in the sidebar. **Stop** ends that session. **Revoke** ends it and disables Computer Use for every bot. Akeru keeps screenshots, typed text, window titles, and application content out of stored runtime events.
+
Akeru enables each installed plugin for every bot by default. Use a bot's **Plugins** setting to disable it for that bot. A provider change starts a fresh provider session with the same enabled plugins.
Ask the agent to add a Custom MCP server when a connector is not in the directory. Existing Custom MCP servers remain under **Installed**, where you can edit, disable, or delete them. Akeru stores no plugin credentials in the directory or MCP registry. Keep credentials in the host environment or the service sign-in flow.
diff --git a/packages/contracts/src/orchestration.ts b/packages/contracts/src/orchestration.ts
index 1731a003c..e12a3a9aa 100644
--- a/packages/contracts/src/orchestration.ts
+++ b/packages/contracts/src/orchestration.ts
@@ -414,6 +414,7 @@ export const OrchestrationSession = Schema.Struct({
providerName: Schema.NullOr(TrimmedNonEmptyString),
providerInstanceId: Schema.optional(ProviderInstanceId),
runtimeMode: RuntimeMode.pipe(Schema.withDecodingDefault(Effect.succeed(DEFAULT_RUNTIME_MODE))),
+ mcpServerIds: Schema.optional(Schema.Array(McpServerId)),
activeTurnId: Schema.NullOr(TurnId),
lastError: Schema.NullOr(TrimmedNonEmptyString),
updatedAt: IsoDateTime,
diff --git a/plugins/catalog.test.ts b/plugins/catalog.test.ts
index a665d6215..d96477572 100644
--- a/plugins/catalog.test.ts
+++ b/plugins/catalog.test.ts
@@ -20,6 +20,7 @@ const EXPECTED_DIRECTORY_IDS = [
"canva",
"cloudflare",
"coda",
+ "computer-use",
"context",
"customer-io",
"datadog",
@@ -165,7 +166,7 @@ describe("plugin catalog loader", () => {
const directory = loadDirectoryCatalog();
const catalog = loadCatalog();
expect(directory.map((plugin) => plugin.id).toSorted()).toEqual(EXPECTED_DIRECTORY_IDS);
- expect(new Set(directory.map((plugin) => plugin.id)).size).toBe(51);
+ expect(new Set(directory.map((plugin) => plugin.id)).size).toBe(52);
expect(catalog.map((plugin) => plugin.id)).toEqual(EXPECTED_INSTALLABLE_IDS);
expect(catalog.map((plugin) => `builtin-${plugin.id}`)).toEqual(
EXPECTED_INSTALLABLE_IDS.map((id) => `builtin-${id}`),
@@ -217,13 +218,13 @@ describe("plugin catalog loader", () => {
const pending = loadDirectoryCatalog().filter(
(plugin) => !EXPECTED_INSTALLABLE_IDS.some((id) => id === plugin.id),
);
- expect(pending).toHaveLength(47);
+ expect(pending).toHaveLength(48);
expect(pending.filter((plugin) => plugin.catalogStatus === "approval-pending")).toHaveLength(
16,
);
expect(
pending.filter((plugin) => plugin.catalogStatus === "verification-pending"),
- ).toHaveLength(31);
+ ).toHaveLength(32);
for (const plugin of pending) {
expect(["approval-pending", "verification-pending"]).toContain(plugin.catalogStatus);
expect(plugin.connection).toMatchObject({
diff --git a/plugins/entries/computer-use/logo-dark.svg b/plugins/entries/computer-use/logo-dark.svg
new file mode 100644
index 000000000..48517141a
--- /dev/null
+++ b/plugins/entries/computer-use/logo-dark.svg
@@ -0,0 +1,5 @@
+<svg width="24" height="24" viewBox="0 0 40 40" xmlns="http://www.w3.org/2000/svg">
+ <circle cx="20" cy="20" r="14" fill="none" stroke="currentColor" stroke-width="3"/>
+ <path d="M20 7v26M8.7 13.5l22.6 13M8.7 26.5l22.6-13" fill="none" stroke="currentColor" stroke-width="3" stroke-linecap="round"/>
+ <circle cx="20" cy="20" r="5" fill="currentColor"/>
+</svg>
diff --git a/plugins/entries/computer-use/logo.svg b/plugins/entries/computer-use/logo.svg
new file mode 100644
index 000000000..48517141a
--- /dev/null
+++ b/plugins/entries/computer-use/logo.svg
@@ -0,0 +1,5 @@
+<svg width="24" height="24" viewBox="0 0 40 40" xmlns="http://www.w3.org/2000/svg">
+ <circle cx="20" cy="20" r="14" fill="none" stroke="currentColor" stroke-width="3"/>
+ <path d="M20 7v26M8.7 13.5l22.6 13M8.7 26.5l22.6-13" fill="none" stroke="currentColor" stroke-width="3" stroke-linecap="round"/>
+ <circle cx="20" cy="20" r="5" fill="currentColor"/>
+</svg>
diff --git a/plugins/entries/computer-use/plugin.json b/plugins/entries/computer-use/plugin.json
new file mode 100644
index 000000000..5bd3975d4
--- /dev/null
+++ b/plugins/entries/computer-use/plugin.json
@@ -0,0 +1,56 @@
+{
+ "schemaVersion": 1,
+ "id": "computer-use",
+ "name": "Codex Computer Use",
+ "description": "Let one Codex bot inspect and control local macOS applications with approval for every action.",
+ "primaryCategory": "Build",
+ "tags": ["codex", "computer-use", "macos", "desktop", "local"],
+ "capabilities": [
+ "capture the screen",
+ "inspect local applications",
+ "click and scroll",
+ "type text and keyboard shortcuts",
+ "open local applications"
+ ],
+ "publisher": { "name": "OpenAI", "url": "https://openai.com" },
+ "maintainer": { "name": "OpenAI", "url": "https://openai.com" },
+ "documentationUrl": "https://developers.openai.com/codex/computer-use",
+ "sourceUrl": "https://developers.openai.com/codex/plugins",
+ "license": "Proprietary",
+ "logo": {
+ "provenance": {
+ "sourceUrl": "https://openai.com/brand/",
+ "license": "OpenAI brand asset"
+ }
+ },
+ "platforms": ["macos"],
+ "transport": {
+ "type": "stdio",
+ "command": "akeru-codex-computer-use",
+ "args": ["mcp"]
+ },
+ "connection": {
+ "type": "verification-pending",
+ "blocker": "A live Akeru Codex tool call still needs verification with a signed-in disposable environment."
+ },
+ "authentication": "none",
+ "requiredCredentials": [],
+ "setup": [
+ "Install the official plugin with codex plugin add computer-use@openai-bundled --json.",
+ "Use a Codex bot on macOS. Screen Recording and Accessibility are requested only when control starts."
+ ],
+ "permissions": [
+ {
+ "id": "inspect-local-desktop",
+ "description": "Capture the screen and inspect local applications in memory.",
+ "approval": "read"
+ },
+ {
+ "id": "control-local-desktop",
+ "description": "Click, scroll, type, use keyboard shortcuts, and open local applications.",
+ "approval": "production"
+ }
+ ],
+ "approvals": ["production"],
+ "catalogStatus": "verification-pending"
+}
diff --git a/plugins/lifecycle-matrix.test.ts b/plugins/lifecycle-matrix.test.ts
index dc55d21be..d3be6926a 100644
--- a/plugins/lifecycle-matrix.test.ts
+++ b/plugins/lifecycle-matrix.test.ts
@@ -9,7 +9,7 @@ import {
} from "./catalog";
const EXPECTED_IDS =
- "ahrefs apify apollo asana atlassian attio canva cloudflare coda context customer-io datadog docusign dropbox exa executor figma firecrawl framer github help-scout hubspot intercom lemon-squeezy linear mobbin monday netlify notion paddle paper parallel-search paypal pipedrive posthog railway render salesforce semrush sentry sequenzy shopify slack stripe superside tavily typefully vercel webflow zendesk zernio".split(
+ "ahrefs apify apollo asana atlassian attio canva cloudflare coda computer-use context customer-io datadog docusign dropbox exa executor figma firecrawl framer github help-scout hubspot intercom lemon-squeezy linear mobbin monday netlify notion paddle paper parallel-search paypal pipedrive posthog railway render salesforce semrush sentry sequenzy shopify slack stripe superside tavily typefully vercel webflow zendesk zernio".split(
" ",
);
const INSTALLABLE_IDS = ["context", "exa", "firecrawl", "parallel-search"];
@@ -35,13 +35,13 @@ describe("milestone 13 plugin lifecycle matrix", () => {
]);
const pending = directory.filter((plugin) => !INSTALLABLE_IDS.includes(plugin.id));
- expect(pending).toHaveLength(47);
+ expect(pending).toHaveLength(48);
expect(pending.filter((plugin) => plugin.catalogStatus === "approval-pending")).toHaveLength(
16,
);
expect(
pending.filter((plugin) => plugin.catalogStatus === "verification-pending"),
- ).toHaveLength(31);
+ ).toHaveLength(32);
for (const plugin of pending) {
expect(["approval-pending", "verification-pending"]).toContain(plugin.catalogStatus);
expect(plugin.connection).toMatchObject({
@@ -66,6 +66,13 @@ describe("milestone 13 plugin lifecycle matrix", () => {
transport: { type: "url", url: "https://api.githubcopilot.com/mcp/" },
connection: { type: "verification-pending" },
});
+ expect(byId.get("computer-use")).toMatchObject({
+ platforms: ["macos"],
+ connection: {
+ type: "verification-pending",
+ blocker: expect.stringContaining("live Akeru Codex tool call"),
+ },
+ });
expect(byId.get("paper")).toMatchObject({
transport: { type: "url", url: "http://127.0.0.1:29979/mcp" },
connection: { type: "verification-pending" },
diff --git a/plugins/schema.ts b/plugins/schema.ts
index 26ac8579a..c924b8e2f 100644
--- a/plugins/schema.ts
+++ b/plugins/schema.ts
@@ -393,7 +393,11 @@ function validateManifest(manifest: PluginManifest): PluginManifest {
`Plugin '${manifest.id}' labels an API key connection without API key authentication.`,
);
}
- if (manifest.transport.type === "stdio" && manifest.connection.type !== "local") {
+ if (
+ manifest.transport.type === "stdio" &&
+ manifest.connection.type !== "local" &&
+ manifest.connection.type !== "verification-pending"
+ ) {
throw new TypeError(`Plugin '${manifest.id}' must label its stdio connection as local.`);
}
if (
diff --git a/scripts/validate-plugin-catalog.test.ts b/scripts/validate-plugin-catalog.test.ts
index 4a8d19401..7e64f1893 100644
--- a/scripts/validate-plugin-catalog.test.ts
+++ b/scripts/validate-plugin-catalog.test.ts
@@ -29,7 +29,7 @@ afterEach(() => {
describe("plugin catalog validation", () => {
it("validates the live plugin directory", () => {
- expect(validatePluginCatalog()).toHaveLength(51);
+ expect(validatePluginCatalog()).toHaveLength(52);
});
it("validates isolated manifests and official local assets", () => {