Open-source Grok Bot alternative for named AI teammates. Local-first desktop, web, and mobile clients with per-bot tools, instructions, and memory. Independent fork of T3 Code.
feat(plugins): add local Codex computer use (#68)
Leo966ed2fc8633
diff --git a/apps/server/src/mcp/PreviewSnapshotRedaction.ts b/apps/server/src/mcp/PreviewSnapshotRedaction.tsindex 31e3cf107..46dc69546 100644--- a/apps/server/src/mcp/PreviewSnapshotRedaction.ts+++ b/apps/server/src/mcp/PreviewSnapshotRedaction.ts@@ -3,7 +3,7 @@ import { PNG } from "pngjs";import { redactSensitiveText } from "./SensitiveDataRedaction.ts";const REDACTED = "[REDACTED]";-const MAX_SCREENSHOT_BYTES = 20 * 1_024 * 1_024;+export const MAX_SCREENSHOT_BYTES = 20 * 1_024 * 1_024;const MAX_SCREENSHOT_PIXELS = 16_000_000;const secretField =/^(?:api[_-]?key|access[_-]?token|refresh[_-]?token|token|secret|password|authorization|cookie|set-cookie|session|sessionId|clientSecret|awsSecretAccessKey)$/i;diff --git a/apps/server/src/orchestration/Layers/ProjectionPipeline.ts b/apps/server/src/orchestration/Layers/ProjectionPipeline.tsindex 76f3811f7..6de74143a 100644--- a/apps/server/src/orchestration/Layers/ProjectionPipeline.ts+++ b/apps/server/src/orchestration/Layers/ProjectionPipeline.ts@@ -1424,6 +1424,7 @@ const makeOrchestrationProjectionPipeline = Effect.fn("makeOrchestrationProjectiproviderName: event.payload.session.providerName,providerInstanceId: event.payload.session.providerInstanceId ?? null,runtimeMode: event.payload.session.runtimeMode,+ mcpServerIds: [...(event.payload.session.mcpServerIds ?? [])],activeTurnId: event.payload.session.activeTurnId,lastError: event.payload.session.lastError,updatedAt: event.payload.session.updatedAt,diff --git a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.test.ts b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.test.tsindex 3db74f6f4..0b1411f90 100644--- a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.test.ts+++ b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.test.ts@@ -391,6 +391,7 @@ projectionSnapshotLayer("ProjectionSnapshotQuery", (it) => {status: "running",providerName: "codex",runtimeMode: "approval-required",+ mcpServerIds: [],activeTurnId: asTurnId("turn-1"),lastError: null,updatedAt: "2026-02-24T00:00:07.000Z",@@ -477,6 +478,7 @@ projectionSnapshotLayer("ProjectionSnapshotQuery", (it) => {status: "running",providerName: "codex",runtimeMode: "approval-required",+ mcpServerIds: [],activeTurnId: asTurnId("turn-1"),lastError: null,updatedAt: "2026-02-24T00:00:07.000Z",diff --git a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.tsindex fbb643060..63235fc3a 100644--- a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts+++ b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts@@ -127,7 +127,9 @@ const ProjectionThreadActivityDbRowSchema = ProjectionThreadActivity.mapFields(sequence: Schema.NullOr(NonNegativeInt),}),);-const ProjectionThreadSessionDbRowSchema = ProjectionThreadSession;+const ProjectionThreadSessionDbRowSchema = ProjectionThreadSession.mapFields(+ Struct.assign({ mcpServerIds: Schema.fromJsonString(Schema.Array(McpServerId)) }),+);const ProjectionCheckpointDbRowSchema = ProjectionCheckpoint.mapFields(Struct.assign({files: Schema.fromJsonString(Schema.Array(OrchestrationCheckpointFile)),@@ -342,6 +344,7 @@ function mapSessionRow(providerName: row.providerName,...(row.providerInstanceId !== null ? { providerInstanceId: row.providerInstanceId } : {}),runtimeMode: row.runtimeMode,+ mcpServerIds: row.mcpServerIds,activeTurnId: row.activeTurnId,lastError: row.lastError,updatedAt: row.updatedAt,@@ -722,6 +725,7 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () {provider_session_id AS "providerSessionId",provider_thread_id AS "providerThreadId",runtime_mode AS "runtimeMode",+ mcp_server_ids_json AS "mcpServerIds",active_turn_id AS "activeTurnId",last_error AS "lastError",updated_at AS "updatedAt"@@ -743,6 +747,7 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () {sessions.provider_session_id AS "providerSessionId",sessions.provider_thread_id AS "providerThreadId",sessions.runtime_mode AS "runtimeMode",+ sessions.mcp_server_ids_json AS "mcpServerIds",sessions.active_turn_id AS "activeTurnId",sessions.last_error AS "lastError",sessions.updated_at AS "updatedAt"@@ -768,6 +773,7 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () {sessions.provider_session_id AS "providerSessionId",sessions.provider_thread_id AS "providerThreadId",sessions.runtime_mode AS "runtimeMode",+ sessions.mcp_server_ids_json AS "mcpServerIds",sessions.active_turn_id AS "activeTurnId",sessions.last_error AS "lastError",sessions.updated_at AS "updatedAt"@@ -1194,6 +1200,7 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () {provider_name AS "providerName",provider_instance_id AS "providerInstanceId",runtime_mode AS "runtimeMode",+ mcp_server_ids_json AS "mcpServerIds",active_turn_id AS "activeTurnId",last_error AS "lastError",updated_at AS "updatedAt"@@ -1816,18 +1823,7 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () {for (const row of sessionRows) {updatedAt = maxIso(updatedAt, row.updatedAt);- sessionsByThread.set(row.threadId, {- threadId: row.threadId,- status: row.status,- providerName: row.providerName,- ...(row.providerInstanceId !== null- ? { providerInstanceId: row.providerInstanceId }- : {}),- runtimeMode: row.runtimeMode,- activeTurnId: row.activeTurnId,- lastError: row.lastError,- updatedAt: row.updatedAt,- });+ sessionsByThread.set(row.threadId, mapSessionRow(row));}const repositoryIdentities = yield* resolveRepositoryIdentitiesForProjects(diff --git a/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts b/apps/server/src/orchestration/Layers/ProviderCommandReactor.tsindex c152a287e..1ec32bac3 100644--- a/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts+++ b/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts@@ -683,6 +683,7 @@ const make = Effect.gen(function* () {providerName: activeSession?.provider ?? preferredProvider,providerInstanceId: activeSession?.providerInstanceId ?? desiredInstanceId,runtimeMode: desiredRuntimeMode,+ mcpServerIds: activeSession?.mcpServerIds ?? [],activeTurnId: null,lastError: null,updatedAt: createdAt,@@ -790,6 +791,7 @@ const make = Effect.gen(function* () {providerName: session.provider,providerInstanceId: session.providerInstanceId,runtimeMode: desiredRuntimeMode,+ mcpServerIds: session.mcpServerIds ?? [],// Provider turn ids are not orchestration turn ids.activeTurnId: null,lastError: session.lastError ?? null,@@ -1664,6 +1666,7 @@ const make = Effect.gen(function* () {? { providerInstanceId: thread.session.providerInstanceId }: {}),runtimeMode: thread.session?.runtimeMode ?? DEFAULT_RUNTIME_MODE,+ mcpServerIds: [],activeTurnId: null,lastError: thread.session?.lastError ?? null,updatedAt: now,diff --git a/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts b/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.tsindex 620e14f12..3d2e3a763 100644--- a/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts+++ b/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts@@ -1774,6 +1774,7 @@ const make = Effect.gen(function* () {? { providerInstanceId: event.providerInstanceId }: {}),runtimeMode: thread.session?.runtimeMode ?? "full-access",+ mcpServerIds: thread.session?.mcpServerIds ?? [],activeTurnId: nextActiveTurnId,lastError,updatedAt: now,@@ -2024,6 +2025,7 @@ const make = Effect.gen(function* () {? { providerInstanceId: event.providerInstanceId }: {}),runtimeMode: thread.session?.runtimeMode ?? "full-access",+ mcpServerIds: thread.session?.mcpServerIds ?? [],activeTurnId: eventTurnId ?? null,lastError: runtimeErrorMessage,updatedAt: now,diff --git a/apps/server/src/persistence/Layers/ProjectionThreadSessions.ts b/apps/server/src/persistence/Layers/ProjectionThreadSessions.tsindex dcb750983..5853442de 100644--- a/apps/server/src/persistence/Layers/ProjectionThreadSessions.ts+++ b/apps/server/src/persistence/Layers/ProjectionThreadSessions.ts@@ -2,6 +2,10 @@ import * as SqlClient from "effect/unstable/sql/SqlClient";import * as SqlSchema from "effect/unstable/sql/SqlSchema";import * as Effect from "effect/Effect";import * as Layer from "effect/Layer";+import * as Schema from "effect/Schema";+import * as Struct from "effect/Struct";++import { McpServerId } from "@t3tools/contracts";import { toPersistenceSqlError } from "../Errors.ts";@@ -13,6 +17,12 @@ import {GetProjectionThreadSessionInput,} from "../Services/ProjectionThreadSessions.ts";+const ProjectionThreadSessionDbRow = ProjectionThreadSession.mapFields(+ Struct.assign({+ mcpServerIds: Schema.fromJsonString(Schema.Array(McpServerId)),+ }),+);+const makeProjectionThreadSessionRepository = Effect.gen(function* () {const sql = yield* SqlClient.SqlClient;@@ -26,6 +36,7 @@ const makeProjectionThreadSessionRepository = Effect.gen(function* () {provider_name,provider_instance_id,runtime_mode,+ mcp_server_ids_json,active_turn_id,last_error,updated_at@@ -36,6 +47,7 @@ const makeProjectionThreadSessionRepository = Effect.gen(function* () {${row.providerName},${row.providerInstanceId},${row.runtimeMode},+ ${JSON.stringify(row.mcpServerIds)},${row.activeTurnId},${row.lastError},${row.updatedAt}@@ -46,6 +58,7 @@ const makeProjectionThreadSessionRepository = Effect.gen(function* () {provider_name = excluded.provider_name,provider_instance_id = excluded.provider_instance_id,runtime_mode = excluded.runtime_mode,+ mcp_server_ids_json = excluded.mcp_server_ids_json,active_turn_id = excluded.active_turn_id,last_error = excluded.last_error,updated_at = excluded.updated_at@@ -54,7 +67,7 @@ const makeProjectionThreadSessionRepository = Effect.gen(function* () {const getProjectionThreadSessionRow = SqlSchema.findOneOption({Request: GetProjectionThreadSessionInput,- Result: ProjectionThreadSession,+ Result: ProjectionThreadSessionDbRow,execute: ({ threadId }) =>sql`SELECT@@ -63,6 +76,7 @@ const makeProjectionThreadSessionRepository = Effect.gen(function* () {provider_name AS "providerName",provider_instance_id AS "providerInstanceId",runtime_mode AS "runtimeMode",+ mcp_server_ids_json AS "mcpServerIds",active_turn_id AS "activeTurnId",last_error AS "lastError",updated_at AS "updatedAt"diff --git a/apps/server/src/persistence/Migrations.ts b/apps/server/src/persistence/Migrations.tsindex ca403b510..ebf57e646 100644--- a/apps/server/src/persistence/Migrations.ts+++ b/apps/server/src/persistence/Migrations.ts@@ -71,6 +71,7 @@ import Migration0055 from "./Migrations/055_AkeruMemoryCandidates.ts";import Migration0056 from "./Migrations/056_AkeruBotUsageLedger.ts";import Migration0057 from "./Migrations/057_AkeruMemoryCandidateUpdates.ts";import Migration0058 from "./Migrations/058_ProjectionDelegations.ts";+import Migration0059 from "./Migrations/059_ProjectionThreadSessionMcpServers.ts";/*** Migration loader with all migrations defined inline.@@ -141,6 +142,7 @@ export const migrationEntries = [[56, "AkeruBotUsageLedger", Migration0056],[57, "AkeruMemoryCandidateUpdates", Migration0057],[58, "ProjectionDelegations", Migration0058],+ [59, "ProjectionThreadSessionMcpServers", Migration0059],] as const;export const migrationManifest = migrationEntries.map(([id, name]) => [id, name] as const);diff --git a/apps/server/src/persistence/Migrations/058_ProjectionDelegations.test.ts b/apps/server/src/persistence/Migrations/058_ProjectionDelegations.test.tsindex 0de251d2f..609958873 100644--- a/apps/server/src/persistence/Migrations/058_ProjectionDelegations.test.ts+++ b/apps/server/src/persistence/Migrations/058_ProjectionDelegations.test.ts@@ -19,7 +19,9 @@ it.layer(Layer.mergeAll(NodeSqliteClient.layerMemory()))("058_ProjectionDelegatiassert.deepEqual(before, []);yield* runMigrations();- assert.deepEqual(migrationManifest.at(-1), [58, "ProjectionDelegations"]);+ assert.isTrue(+ migrationManifest.some(([id, name]) => id === 58 && name === "ProjectionDelegations"),+ );yield* sql`INSERT INTO projection_delegations (delegation_id, record_json)diff --git a/apps/server/src/persistence/Migrations/059_ProjectionThreadSessionMcpServers.test.ts b/apps/server/src/persistence/Migrations/059_ProjectionThreadSessionMcpServers.test.tsnew file mode 100644index 000000000..67a14c478--- /dev/null+++ b/apps/server/src/persistence/Migrations/059_ProjectionThreadSessionMcpServers.test.ts@@ -0,0 +1,29 @@+import { assert, it } from "@effect/vitest";+import * as Effect from "effect/Effect";+import * as Layer from "effect/Layer";+import * as SqlClient from "effect/unstable/sql/SqlClient";++import { runMigrations } from "../Migrations.ts";+import * as NodeSqliteClient from "../NodeSqliteClient.ts";++it.layer(Layer.mergeAll(NodeSqliteClient.layerMemory()))(+ "059_ProjectionThreadSessionMcpServers",+ (it) => {+ it.effect("adds an empty MCP server list to projected sessions", () =>+ Effect.gen(function* () {+ const sql = yield* SqlClient.SqlClient;+ yield* runMigrations({ toMigrationInclusive: 57 });+ yield* runMigrations({ toMigrationInclusive: 59 });++ const columns = yield* sql<{+ readonly name: string;+ readonly notnull: number;+ readonly dflt_value: string | null;+ }>`PRAGMA table_info(projection_thread_sessions)`;+ const mcpServerIds = columns.find((column) => column.name === "mcp_server_ids_json");+ assert.equal(mcpServerIds?.notnull, 1);+ assert.equal(mcpServerIds?.dflt_value, "'[]'");+ }),+ );+ },+);diff --git a/apps/server/src/persistence/Migrations/059_ProjectionThreadSessionMcpServers.ts b/apps/server/src/persistence/Migrations/059_ProjectionThreadSessionMcpServers.tsnew file mode 100644index 000000000..81bb2706c--- /dev/null+++ b/apps/server/src/persistence/Migrations/059_ProjectionThreadSessionMcpServers.ts@@ -0,0 +1,15 @@+import * as Effect from "effect/Effect";+import * as SqlClient from "effect/unstable/sql/SqlClient";++export default Effect.gen(function* () {+ const sql = yield* SqlClient.SqlClient;+ const columns = yield* sql<{ readonly name: string }>`+ PRAGMA table_info(projection_thread_sessions)+ `;+ if (!columns.some((column) => column.name === "mcp_server_ids_json")) {+ yield* sql`+ ALTER TABLE projection_thread_sessions+ ADD COLUMN mcp_server_ids_json TEXT NOT NULL DEFAULT '[]'+ `;+ }+});diff --git a/apps/server/src/persistence/Services/ProjectionThreadSessions.ts b/apps/server/src/persistence/Services/ProjectionThreadSessions.tsindex 0510d8ab3..4c0bcb17a 100644--- a/apps/server/src/persistence/Services/ProjectionThreadSessions.ts+++ b/apps/server/src/persistence/Services/ProjectionThreadSessions.ts@@ -9,6 +9,7 @@import {RuntimeMode,IsoDateTime,+ McpServerId,OrchestrationSessionStatus,ProviderInstanceId,ThreadId,@@ -27,6 +28,7 @@ export const ProjectionThreadSession = Schema.Struct({providerName: Schema.NullOr(Schema.String),providerInstanceId: Schema.NullOr(ProviderInstanceId),runtimeMode: RuntimeMode,+ mcpServerIds: Schema.Array(McpServerId),activeTurnId: Schema.NullOr(TurnId),lastError: Schema.NullOr(Schema.String),updatedAt: IsoDateTime,diff --git a/apps/server/src/provider/AkeruMastraHarness.ts b/apps/server/src/provider/AkeruMastraHarness.tsindex 78f186559..f193e6cd5 100644--- a/apps/server/src/provider/AkeruMastraHarness.ts+++ b/apps/server/src/provider/AkeruMastraHarness.ts@@ -40,6 +40,7 @@ import { AKERU_AGENT_INSTRUCTIONS, AKERU_BOT_INSTRUCTIONS } from "./AkeruAgentInimport { akeruKimiProvider, type AkeruKimiAccess } from "./AkeruKimiProvider.ts";import { createAkeruMastraTools } from "./AkeruMastraTools.ts";import type { AkeruToolRuntime } from "./AkeruToolRuntime.ts";+import { isCodexComputerUseTool } from "./CodexComputerUse.ts";const DEFAULT_MODEL_ID = "openai/gpt-5.6-sol";const decodeProductFeedbackToolDraft = Schema.decodeUnknownExit(ProductFeedbackToolDraft, {@@ -363,7 +364,8 @@ function approvalAwareTools(};const existing = configured.needsApprovalFn ?? configured.requireApproval;const needsApproval: NeedsApprovalFn = async (input, context) => {- const protectedAction = akeruActionNeedsApproval(name, input);+ const protectedAction =+ isCodexComputerUseTool(name) || akeruActionNeedsApproval(name, input);await options.syncThreadToolApproval?.(threadId, name, protectedAction);return (protectedAction ||diff --git a/apps/server/src/provider/AkeruSessionResources.test.ts b/apps/server/src/provider/AkeruSessionResources.test.tsindex 53fdbaea2..714d62490 100644--- a/apps/server/src/provider/AkeruSessionResources.test.ts+++ b/apps/server/src/provider/AkeruSessionResources.test.ts@@ -8,6 +8,7 @@ import { McpServerId } from "@t3tools/contracts";import { afterEach, describe, expect, it, vi } from "vite-plus/test";import { AkeruSessionResources } from "./AkeruSessionResources.ts";+import { CODEX_COMPUTER_USE_SERVER_ID } from "./CodexComputerUse.ts";import {type AkeruBotWorkspace,type AkeruRemoteSession,@@ -50,6 +51,37 @@ function browser(overrides?: { reconnect?: () => Promise<void>; close?: () => Pr};}+function computerServer() {+ return {+ id: CODEX_COMPUTER_USE_SERVER_ID as never,+ name: "Computer Use",+ transport: "stdio" as const,+ command: "akeru-codex-computer-use",+ enabled: true,+ createdAt: "2026-01-01T00:00:00.000Z",+ updatedAt: "2026-01-01T00:00:00.000Z",+ };+}++function mcpManager(+ status: { connected: boolean; toolCount: number; error?: string },+ tools: Record<string, unknown> = {},+) {+ return {+ init: vi.fn(async () => undefined),+ disconnect: vi.fn(async () => undefined),+ getTools: vi.fn(() => tools),+ getServerStatuses: vi.fn(() => [+ {+ name: CODEX_COMPUTER_USE_SERVER_ID,+ transport: "stdio" as const,+ toolNames: status.toolCount > 0 ? [`${CODEX_COMPUTER_USE_SERVER_ID}_control`] : [],+ ...status,+ },+ ]),+ };+}+const remoteInput = {resourceScope: "shared",workspaceResourceKey: "vercel:shared",@@ -386,6 +418,67 @@ describe("AkeruSessionResources", () => {await resources.shutdown();});+ it("allows one Computer Use controller and releases it on stop", async () => {+ const manager = mcpManager({ connected: true, toolCount: 1 });+ const resources = new AkeruSessionResources({+ stateDir: stateDir(),+ hostPlatform: "darwin",+ makeRemoteWorkspace: async () => workspace(),+ makeBotBrowser: () => browser(),+ makeMcpManager: () => manager as never,+ resolveComputerUseServer: async () => ({+ command: "/local/launcher",+ args: ["mcp"],+ env: {},+ }),+ toMcpServerConfigs: () => ({+ [CODEX_COMPUTER_USE_SERVER_ID]: { command: "sentinel" },+ }),+ });+ const input = { ...remoteInput, mcpServers: [computerServer()] };++ await resources.acquire({ ...input, threadId: "controller" });+ await expect(resources.acquire({ ...input, threadId: "blocked" })).rejects.toThrow(+ "already controlled",+ );+ await resources.release("controller");+ await resources.acquire({ ...input, threadId: "replacement" });+ await resources.shutdown();+ });++ it("redacts Computer Use results at the MCP tool boundary", async () => {+ const execute = vi.fn(async () => ({ screenshot: { url: "https://example.com/frame.png" } }));+ const toolName = `${CODEX_COMPUTER_USE_SERVER_ID}_control`;+ const manager = mcpManager({ connected: true, toolCount: 1 }, { [toolName]: { execute } });+ const resources = new AkeruSessionResources({+ stateDir: stateDir(),+ hostPlatform: "darwin",+ makeRemoteWorkspace: async () => workspace(),+ makeBotBrowser: () => browser(),+ makeMcpManager: () => manager as never,+ resolveComputerUseServer: async () => ({+ command: "/local/launcher",+ args: ["mcp"],+ env: {},+ }),+ toMcpServerConfigs: () => ({+ [CODEX_COMPUTER_USE_SERVER_ID]: { command: "sentinel" },+ }),+ });++ await resources.acquire({+ ...remoteInput,+ threadId: "controller",+ mcpServers: [computerServer()],+ });+ const tool = Reflect.get(resources.getConnectorTools("controller"), toolName) as {+ execute: () => Promise<unknown>;+ };+ await expect(tool.execute()).rejects.toThrow("unknown screenshot");+ expect(execute).toHaveBeenCalledOnce();+ await resources.shutdown();+ });+it("creates and attaches a browser for remote workspaces", async () => {const browserEndpoint = vi.fn(async () => ({url: "https://browser.example",@@ -443,4 +536,67 @@ describe("AkeruSessionResources", () => {expect(resources.getConnectorTools("remote-mcp")).toEqual({ exa_search: {} });await resources.shutdown();});++ it("releases the Computer Use lock when MCP health fails", async () => {+ const failed = mcpManager({ connected: false, toolCount: 0, error: "Accessibility denied" });+ const healthy = mcpManager({ connected: true, toolCount: 1 });+ const makeMcpManager = vi+ .fn()+ .mockReturnValueOnce(failed as never)+ .mockReturnValueOnce(healthy as never);+ const resources = new AkeruSessionResources({+ stateDir: stateDir(),+ hostPlatform: "darwin",+ makeRemoteWorkspace: async () => workspace(),+ makeBotBrowser: () => browser(),+ makeMcpManager,+ resolveComputerUseServer: async () => ({+ command: "/local/launcher",+ args: ["mcp"],+ env: {},+ }),+ toMcpServerConfigs: () => ({+ [CODEX_COMPUTER_USE_SERVER_ID]: { command: "sentinel" },+ }),+ });+ const input = { ...remoteInput, mcpServers: [computerServer()] };++ await expect(resources.acquire({ ...input, threadId: "failed" })).rejects.toThrow(+ "needs Screen Recording and Accessibility permissions",+ );+ await resources.acquire({ ...input, threadId: "replacement" });+ await resources.shutdown();+ });++ it("does not expose raw MCP startup errors", async () => {+ const failed = mcpManager({+ connected: false,+ toolCount: 0,+ error: "Failed at /private/tester/Secret App",+ });+ const resources = new AkeruSessionResources({+ stateDir: stateDir(),+ hostPlatform: "darwin",+ makeRemoteWorkspace: async () => workspace(),+ makeBotBrowser: () => browser(),+ makeMcpManager: () => failed as never,+ resolveComputerUseServer: async () => ({+ command: "/local/launcher",+ args: ["mcp"],+ env: {},+ }),+ toMcpServerConfigs: () => ({+ [CODEX_COMPUTER_USE_SERVER_ID]: { command: "sentinel" },+ }),+ });++ await expect(+ resources.acquire({+ ...remoteInput,+ threadId: "failed",+ mcpServers: [computerServer()],+ }),+ ).rejects.toThrow("Computer Use MCP failed to start.");+ await resources.shutdown();+ });});diff --git a/apps/server/src/provider/AkeruSessionResources.ts b/apps/server/src/provider/AkeruSessionResources.tsindex 88253e910..e3805e664 100644--- a/apps/server/src/provider/AkeruSessionResources.ts+++ b/apps/server/src/provider/AkeruSessionResources.ts@@ -1,4 +1,5 @@// @effect-diagnostics nodeBuiltinImport:off+import * as NodeFS from "node:fs";import * as NodePath from "node:path";import type { ToolsInput } from "@mastra/core/agent";@@ -23,6 +24,13 @@ import {type CreateRemoteBotWorkspaceInput,} from "./botWorkspace.ts";import { BotWorkspacePool, type BotWorkspaceLease } from "./botWorkspacePool.ts";+import {+ CODEX_COMPUTER_USE_SERVER_ID,+ isCodexComputerUseServer,+ isCodexComputerUseTool,+ resolveCodexComputerUseServer,+ sanitizeCodexComputerUseResult,+} from "./CodexComputerUse.ts";export interface AkeruSessionResourceInput {readonly threadId: string;@@ -46,6 +54,8 @@ export interface AkeruSessionResourcesOptions {input: CreateRemoteBotWorkspaceInput,) => Promise<AkeruBotWorkspace | Workspace>;readonly makeBotBrowser?: (input: CreateBotBrowserInput) => BotBrowser;+ readonly hostPlatform?: NodeJS.Platform;+ readonly resolveComputerUseServer?: typeof resolveCodexComputerUseServer;readonly onMcpServerConnectionFailure?: (serverId: McpServer["id"]) => void;readonly toMcpServerConfigs: (servers: readonly McpServer[],@@ -66,6 +76,8 @@ export class AkeruSessionResources {private readonly browserReferences = new Map<string, number>();private readonly browserDestroyRequests = new Set<string>();private readonly browserReconnects = new Map<string, Promise<void>>();+ private readonly computerUseTemporaryDirectories = new Map<string, string>();+ private controllingThreadId: string | undefined;private shuttingDown = false;constructor(options: AkeruSessionResourcesOptions) {@@ -95,7 +107,18 @@ export class AkeruSessionResources {private async acquireOnce(input: AkeruSessionResourceInput): Promise<AkeruSessionResourceView> {const key = input.threadId;+ const usesComputer = input.mcpServers.some((server) =>+ isCodexComputerUseServer(String(server.id)),+ );try {+ if (usesComputer) {+ if (this.controllingThreadId && this.controllingThreadId !== key) {+ throw new Error(+ `Computer Use is already controlled by thread '${this.controllingThreadId}'. Stop it before starting another controller.`,+ );+ }+ this.controllingThreadId = key;+ }const workspaceLease = await this.workspacePool.acquire(input.workspaceResourceKey,async () => {@@ -182,10 +205,24 @@ export class AkeruSessionResources {if (input.mcpServers.length > 0) {const attachment = await browser.attachment();+ const configs = this.options.toMcpServerConfigs(input.mcpServers, attachment);+ if (usesComputer) {+ if (!this.options.hostPlatform) {+ throw new Error("Computer Use host platform is unavailable.");+ }+ const config = await (+ this.options.resolveComputerUseServer ?? resolveCodexComputerUseServer+ )({ platform: this.options.hostPlatform });+ configs[CODEX_COMPUTER_USE_SERVER_ID] = config;+ const temporaryDirectory = config.env?.TMPDIR;+ if (typeof temporaryDirectory === "string") {+ this.computerUseTemporaryDirectories.set(key, temporaryDirectory);+ }+ }const manager = (this.options.makeMcpManager ?? createMcpManager)(NodePath.join(this.options.stateDir, "bot-mcp-runtime"),".akeru-runtime",- this.options.toMcpServerConfigs(input.mcpServers, attachment),+ configs,);this.mcpManagers.set(key, manager);try {@@ -194,6 +231,10 @@ export class AkeruSessionResources {for (const server of input.mcpServers) {this.options.onMcpServerConnectionFailure?.(server.id);}+ if (usesComputer) {+ // eslint-disable-next-line preserve-caught-error -- MCP errors may contain private desktop paths.+ throw new Error("Computer Use MCP failed to start.");+ }throw cause;}const serversById = new Map(input.mcpServers.map((server) => [String(server.id), server]));@@ -202,6 +243,17 @@ export class AkeruSessionResources {const server = serversById.get(status.name);if (server) this.options.onMcpServerConnectionFailure?.(server.id);}+ if (usesComputer) {+ const status = manager+ .getServerStatuses()+ .find((server) => server.name === CODEX_COMPUTER_USE_SERVER_ID);+ if (!status?.connected || status.toolCount === 0) {+ if (/screen recording|accessibility/i.test(status?.error ?? "")) {+ throw new Error("Computer Use needs Screen Recording and Accessibility permissions.");+ }+ throw new Error("Computer Use MCP failed to start.");+ }+ }}return {@@ -216,10 +268,31 @@ export class AkeruSessionResources {}getConnectorTools(threadId: string): ToolsInput {- return {+ const tools: ToolsInput = {...this.threadBrowsers.get(threadId)?.tools,...this.mcpManagers.get(threadId)?.getTools(),};+ return Object.fromEntries(+ Object.entries(tools).map(([name, tool]) => {+ const execute = Reflect.get(tool, "execute") as unknown;+ if (!isCodexComputerUseTool(name) || typeof execute !== "function") {+ return [name, tool];+ }+ return [+ name,+ {+ ...tool,+ execute: async (...args: readonly unknown[]) => {+ const temporaryDirectory = this.computerUseTemporaryDirectories.get(threadId);+ return sanitizeCodexComputerUseResult(+ await Reflect.apply(execute, tool, args),+ temporaryDirectory ? { temporaryDirectory } : undefined,+ );+ },+ },+ ];+ }),+ );}getMcpManager(threadId: string): McpManager | undefined {@@ -246,6 +319,15 @@ export class AkeruSessionResources {const manager = this.mcpManagers.get(threadId);this.mcpManagers.delete(threadId);if (manager) await manager.disconnect().catch((cause) => failures.push(cause));+ const computerUseTemporaryDirectory = this.computerUseTemporaryDirectories.get(threadId);+ this.computerUseTemporaryDirectories.delete(threadId);+ if (computerUseTemporaryDirectory) {+ try {+ NodeFS.rmSync(computerUseTemporaryDirectory, { recursive: true, force: true });+ } catch (cause) {+ failures.push(cause);+ }+ }const browser = this.threadBrowsers.get(threadId);const resourceKey = this.browserResourceKeys.get(threadId);@@ -281,6 +363,7 @@ export class AkeruSessionResources {if (userComputerLease) {await userComputerLease.release(options).catch((cause) => failures.push(cause));}+ if (this.controllingThreadId === threadId) this.controllingThreadId = undefined;if (failures.length > 0) throw failures[0];}diff --git a/apps/server/src/provider/CodexComputerUse.test.ts b/apps/server/src/provider/CodexComputerUse.test.tsnew file mode 100644index 000000000..942a1a386--- /dev/null+++ b/apps/server/src/provider/CodexComputerUse.test.ts@@ -0,0 +1,222 @@+// @effect-diagnostics nodeBuiltinImport:off+import * as NodeFS from "node:fs";+import * as NodeOS from "node:os";+import * as NodePath from "node:path";+import * as NodeURL from "node:url";++import { PNG } from "pngjs";+import { afterEach, describe, expect, it } from "vite-plus/test";++import {+ resolveCodexComputerUseServer,+ sanitizeCodexComputerUseResult,+} from "./CodexComputerUse.ts";++const directories = new Set<string>();++function plugin(command = "./bin/launcher") {+ const root = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "akeru-computer-use-"));+ directories.add(root);+ NodeFS.mkdirSync(NodePath.join(root, "bin"));+ NodeFS.writeFileSync(NodePath.join(root, "bin", "launcher"), "#!/bin/sh\n", { mode: 0o700 });+ NodeFS.writeFileSync(+ NodePath.join(root, ".mcp.json"),+ JSON.stringify({+ mcpServers: {+ "computer-use": {+ command,+ args: ["mcp"],+ cwd: ".",+ env_vars: ["CODEX_HOME"],+ },+ },+ }),+ );+ return root;+}++function listing(root: string, overrides?: Record<string, unknown>) {+ return JSON.stringify({+ installed: [+ {+ pluginId: "computer-use@openai-bundled",+ installed: true,+ enabled: true,+ source: { source: "local", path: root },+ ...overrides,+ },+ ],+ available: [],+ });+}++describe("Codex Computer Use resolver", () => {+ afterEach(() => {+ for (const directory of directories) {+ NodeFS.rmSync(directory, { recursive: true, force: true });+ }+ directories.clear();+ });++ it("resolves the enabled official launcher without persisting its path", async () => {+ const root = plugin();+ const result = await resolveCodexComputerUseServer({+ platform: "darwin",+ run: async () => listing(root),+ });++ expect(result).toEqual({+ command: NodeFS.realpathSync(NodePath.join(root, "bin", "launcher")),+ args: ["mcp"],+ env: {+ HOME: NodeOS.homedir(),+ ...(process.env.CODEX_HOME ? { CODEX_HOME: process.env.CODEX_HOME } : {}),+ TMPDIR: expect.stringMatching(/akeru-computer-use-/),+ },+ });+ if (typeof result.env?.TMPDIR === "string") directories.add(result.env.TMPDIR);+ });++ it("fails closed for unsupported, missing, disabled, and malformed plugins", async () => {+ await expect(+ resolveCodexComputerUseServer({ platform: "linux", run: async () => "" }),+ ).rejects.toThrow("only on macOS");+ await expect(+ resolveCodexComputerUseServer({+ platform: "darwin",+ run: async () => JSON.stringify({ installed: [], available: [] }),+ }),+ ).rejects.toThrow("codex plugin add");+ const root = plugin();+ await expect(+ resolveCodexComputerUseServer({+ platform: "darwin",+ run: async () => listing(root, { enabled: false }),+ }),+ ).rejects.toThrow("Enable");+ await expect(+ resolveCodexComputerUseServer({ platform: "darwin", run: async () => "not json" }),+ ).rejects.toThrow("invalid plugin list");+ });++ it("rejects a launcher outside the plugin directory", async () => {+ const root = plugin();+ const external = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "akeru-external-"));+ directories.add(external);+ const externalLauncher = NodePath.join(external, "launcher");+ NodeFS.writeFileSync(externalLauncher, "#!/bin/sh\n", { mode: 0o700 });+ NodeFS.rmSync(NodePath.join(root, "bin", "launcher"));+ NodeFS.symlinkSync(externalLauncher, NodePath.join(root, "bin", "launcher"));+ await expect(+ resolveCodexComputerUseServer({+ platform: "darwin",+ run: async () => listing(root),+ }),+ ).rejects.toThrow("escapes its plugin directory");+ });++ it("rejects a manifest outside the plugin directory", async () => {+ const root = plugin();+ const external = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "akeru-external-"));+ directories.add(external);+ const externalManifest = NodePath.join(external, ".mcp.json");+ NodeFS.renameSync(NodePath.join(root, ".mcp.json"), externalManifest);+ NodeFS.symlinkSync(externalManifest, NodePath.join(root, ".mcp.json"));++ await expect(+ resolveCodexComputerUseServer({+ platform: "darwin",+ run: async () => listing(root),+ }),+ ).rejects.toThrow("manifest escapes its plugin directory");+ });++ it("validates in-memory frames and removes temporary originals", () => {+ const directory = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "akeru-frame-"));+ directories.add(directory);+ const path = NodePath.join(directory, "frame.png");+ const png = new PNG({ width: 1, height: 1 });+ png.data.fill(255);+ NodeFS.writeFileSync(path, PNG.sync.write(png));++ const result = sanitizeCodexComputerUseResult(+ {+ screenshot: { url: NodeURL.pathToFileURL(path).href },+ },+ { temporaryDirectory: directory },+ );++ expect(result).toMatchObject({+ screenshot: { url: expect.stringMatching(/^data:image\/png/) },+ });+ const frameUrl = (result as { screenshot: { url: string } }).screenshot.url;+ const frame = PNG.sync.read(+ Buffer.from(frameUrl.slice("data:image/png;base64,".length), "base64"),+ );+ expect([...frame.data]).toEqual([0, 0, 0, 255]);+ expect(NodeFS.existsSync(path)).toBe(false);++ const inline = sanitizeCodexComputerUseResult({+ type: "image",+ mimeType: "image/png",+ data: PNG.sync.write(png).toString("base64"),+ }) as { data: string };+ expect([...PNG.sync.read(Buffer.from(inline.data, "base64")).data]).toEqual([0, 0, 0, 255]);++ const dataUrl = sanitizeCodexComputerUseResult({+ screenshot: { url: `data:image/png;base64,${PNG.sync.write(png).toString("base64")}` },+ }) as { screenshot: { url: string } };+ expect([+ ...PNG.sync.read(+ Buffer.from(dataUrl.screenshot.url.slice("data:image/png;base64,".length), "base64"),+ ).data,+ ]).toEqual([0, 0, 0, 255]);++ const missingPath = NodePath.join(directory, "private-window-title.png");+ expect(() =>+ sanitizeCodexComputerUseResult(+ {+ screenshot: { url: NodeURL.pathToFileURL(missingPath).href },+ },+ { temporaryDirectory: directory },+ ),+ ).toThrow("unreadable screenshot");+ try {+ sanitizeCodexComputerUseResult(+ {+ screenshot: { url: NodeURL.pathToFileURL(missingPath).href },+ },+ { temporaryDirectory: directory },+ );+ } catch (cause) {+ expect(String(cause)).not.toContain(missingPath);+ }++ expect(() =>+ sanitizeCodexComputerUseResult({+ type: "image",+ mimeType: "image/jpeg",+ data: "invalid",+ }),+ ).toThrow("unsupported screenshot format");+ expect(() =>+ sanitizeCodexComputerUseResult({+ type: "image",+ mimeType: "image/png",+ data: "A".repeat(28 * 1_024 * 1_024),+ }),+ ).toThrow("oversized screenshot");++ const foreignDirectory = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "foreign-frame-"));+ directories.add(foreignDirectory);+ const foreignPath = NodePath.join(foreignDirectory, "frame.png");+ NodeFS.writeFileSync(foreignPath, PNG.sync.write(png));+ expect(() =>+ sanitizeCodexComputerUseResult(+ { screenshot: { url: NodeURL.pathToFileURL(foreignPath).href } },+ { temporaryDirectory: directory },+ ),+ ).toThrow("outside its temporary directory");+ expect(NodeFS.existsSync(foreignPath)).toBe(true);+ });+});diff --git a/apps/server/src/provider/CodexComputerUse.ts b/apps/server/src/provider/CodexComputerUse.tsnew file mode 100644index 000000000..90e807250--- /dev/null+++ b/apps/server/src/provider/CodexComputerUse.ts@@ -0,0 +1,243 @@+// @effect-diagnostics nodeBuiltinImport:off+import * as NodeChildProcess from "node:child_process";+import * as NodeFS from "node:fs";+import * as NodeOS from "node:os";+import * as NodePath from "node:path";+import * as NodeURL from "node:url";+import * as NodeUtil from "node:util";++import type { McpServerConfig } from "@mastra/code-sdk/mcp/index";++import { MAX_SCREENSHOT_BYTES, redactComputerScreenshot } from "../mcp/PreviewSnapshotRedaction.ts";++export const CODEX_COMPUTER_USE_SERVER_ID = "builtin-computer-use";+const CODEX_COMPUTER_USE_PLUGIN_ID = "computer-use@openai-bundled";+const MAX_SCREENSHOT_BASE64_LENGTH = Math.ceil(MAX_SCREENSHOT_BYTES / 3) * 4;+const execFile = NodeUtil.promisify(NodeChildProcess.execFile);++type RunCodex = (args: readonly string[]) => Promise<string>;+export interface CodexComputerUseServerConfig {+ readonly command: string;+ readonly args: string[];+ readonly env: Record<string, string>;+}++function object(value: unknown, label: string): Record<string, unknown> {+ if (typeof value !== "object" || value === null || Array.isArray(value)) {+ throw new Error(`${label} is invalid.`);+ }+ return value as Record<string, unknown>;+}++function entries(value: unknown): readonly Record<string, unknown>[] {+ if (!Array.isArray(value)) throw new Error("Codex returned an invalid plugin list.");+ return value.map((entry) => object(entry, "Codex plugin entry"));+}++function string(value: unknown, label: string): string {+ if (typeof value !== "string" || value.length === 0) throw new Error(`${label} is invalid.`);+ return value;+}++async function runCodex(args: readonly string[]): Promise<string> {+ const result = await execFile("codex", [...args], {+ encoding: "utf8",+ timeout: 10_000,+ killSignal: "SIGKILL",+ maxBuffer: 2 * 1_024 * 1_024,+ });+ return result.stdout;+}++export function isCodexComputerUseServer(id: string): boolean {+ return id === CODEX_COMPUTER_USE_SERVER_ID;+}++export function isCodexComputerUseTool(name: string): boolean {+ return name.startsWith(`${CODEX_COMPUTER_USE_SERVER_ID}_`);+}++function isWithin(parent: string, child: string): boolean {+ const relative = NodePath.relative(parent, child);+ return relative === "" || (!relative.startsWith("..") && !NodePath.isAbsolute(relative));+}++function screenshotDataUrl(url: string, temporaryDirectory?: string): string {+ if (url.startsWith("data:image/png;base64,")) {+ const encoded = url.slice("data:image/png;base64,".length);+ if (encoded.length > MAX_SCREENSHOT_BASE64_LENGTH) {+ throw new Error("Computer Use returned an oversized screenshot.");+ }+ const redacted = redactComputerScreenshot({+ mediaType: "image/png",+ data: Buffer.from(encoded, "base64"),+ });+ return `data:image/png;base64,${Buffer.from(redacted.data).toString("base64")}`;+ }+ if (!url.startsWith("file:")) throw new Error("Computer Use returned an unknown screenshot.");+ if (!temporaryDirectory) throw new Error("Computer Use screenshot cleanup is unavailable.");+ let path: string;+ let ownedDirectory: string;+ try {+ path = NodeFS.realpathSync(NodeURL.fileURLToPath(url));+ ownedDirectory = NodeFS.realpathSync(temporaryDirectory);+ } catch {+ throw new Error("Computer Use returned an unreadable screenshot.");+ }+ if (!isWithin(ownedDirectory, path)) {+ throw new Error("Computer Use returned a screenshot outside its temporary directory.");+ }+ let bytes: Buffer | undefined;+ try {+ const redacted = redactComputerScreenshot({+ mediaType: "image/png",+ data: NodeFS.readFileSync(path),+ });+ bytes = Buffer.from(redacted.data);+ } catch {+ bytes = undefined;+ }+ try {+ NodeFS.rmSync(path);+ } catch {+ throw new Error("Could not remove the temporary Computer Use screenshot.");+ }+ if (!bytes) throw new Error("Computer Use returned an invalid screenshot.");+ return `data:image/png;base64,${bytes.toString("base64")}`;+}++export function sanitizeCodexComputerUseResult(+ value: unknown,+ options?: { readonly temporaryDirectory?: string },+): unknown {+ let remaining = 10_000;+ const visit = (entry: unknown, field?: string): unknown => {+ remaining -= 1;+ if (remaining < 0) throw new Error("Computer Use returned too much data.");+ if (Array.isArray(entry)) return entry.map((item) => visit(item));+ if (typeof entry !== "object" || entry === null) {+ if (field === "screenshot" && entry !== null) {+ throw new Error("Computer Use returned an invalid screenshot.");+ }+ return entry;+ }+ const object = entry as Record<string, unknown>;+ if (object.type === "image") {+ const mediaType = object.mimeType ?? object.mediaType;+ if (mediaType !== "image/png" || typeof object.data !== "string") {+ throw new Error("Computer Use returned an unsupported screenshot format.");+ }+ if (object.data.length > MAX_SCREENSHOT_BASE64_LENGTH) {+ throw new Error("Computer Use returned an oversized screenshot.");+ }+ const redacted = redactComputerScreenshot({+ mediaType: "image/png",+ data: Buffer.from(object.data, "base64"),+ });+ return { ...object, data: Buffer.from(redacted.data).toString("base64") };+ }+ if (field === "screenshot") {+ if (typeof object.url !== "string") {+ throw new Error("Computer Use returned an invalid screenshot.");+ }+ return { ...object, url: screenshotDataUrl(object.url, options?.temporaryDirectory) };+ }+ return Object.fromEntries(Object.entries(object).map(([key, item]) => [key, visit(item, key)]));+ };+ return visit(value);+}++export async function resolveCodexComputerUseServer(options: {+ readonly platform: NodeJS.Platform;+ readonly run?: RunCodex;+}): Promise<CodexComputerUseServerConfig> {+ if (options.platform !== "darwin") {+ throw new Error("Codex Computer Use is supported only on macOS.");+ }++ let output: string;+ try {+ output = await (options.run ?? runCodex)([+ "plugin",+ "list",+ "--marketplace",+ "openai-bundled",+ "--available",+ "--json",+ ]);+ } catch {+ throw new Error("Could not inspect the Codex Computer Use plugin.");+ }++ let document: Record<string, unknown>;+ try {+ document = object(JSON.parse(output), "Codex plugin list");+ } catch {+ throw new Error("Codex returned an invalid plugin list.");+ }+ const plugin = [...entries(document.installed), ...entries(document.available)].find(+ (entry) => entry.pluginId === CODEX_COMPUTER_USE_PLUGIN_ID,+ );+ if (!plugin || plugin.installed !== true) {+ throw new Error(+ "Install the official Codex Computer Use plugin with `codex plugin add computer-use@openai-bundled --json`.",+ );+ }+ if (plugin.enabled !== true) throw new Error("Enable the official Codex Computer Use plugin.");++ const source = object(plugin.source, "Codex Computer Use plugin source");+ if (source.source !== "local") throw new Error("Codex Computer Use must use a local plugin.");+ let root: string;+ let manifestPath: string;+ let manifest: Record<string, unknown>;+ try {+ root = NodeFS.realpathSync(string(source.path, "Codex Computer Use plugin path"));+ manifestPath = NodeFS.realpathSync(NodePath.join(root, ".mcp.json"));+ if (!isWithin(root, manifestPath)) {+ throw new Error("Computer Use MCP manifest escapes its plugin directory.");+ }+ manifest = object(JSON.parse(NodeFS.readFileSync(manifestPath, "utf8")), "MCP manifest");+ } catch (cause) {+ if (cause instanceof Error && cause.message.includes("escapes its plugin directory")) {+ throw cause;+ }+ // eslint-disable-next-line preserve-caught-error -- Plugin paths must not reach persisted errors.+ throw new Error("Could not read the Codex Computer Use plugin.");+ }+ const servers = object(manifest.mcpServers, "MCP servers");+ const server = object(servers["computer-use"], "Computer Use MCP server");+ const command = string(server.command, "Computer Use MCP command");+ if (NodePath.isAbsolute(command) || server.cwd !== ".") {+ throw new Error("Computer Use MCP must use its plugin-relative launcher.");+ }+ if (+ !Array.isArray(server.args) ||+ server.args.length !== 1 ||+ server.args[0] !== "mcp" ||+ !Array.isArray(server.env_vars) ||+ server.env_vars.length !== 1 ||+ server.env_vars[0] !== "CODEX_HOME"+ ) {+ throw new Error("Computer Use MCP has an unsupported launch configuration.");+ }++ let launcher: string;+ try {+ launcher = NodeFS.realpathSync(NodePath.resolve(root, command));+ NodeFS.accessSync(launcher, NodeFS.constants.X_OK);+ } catch {+ throw new Error("Could not start the Codex Computer Use plugin.");+ }+ if (!isWithin(root, launcher)) {+ throw new Error("Computer Use MCP launcher escapes its plugin directory.");+ }+ return {+ command: launcher,+ args: ["mcp"],+ env: {+ HOME: NodeOS.homedir(),+ ...(process.env.CODEX_HOME ? { CODEX_HOME: process.env.CODEX_HOME } : {}),+ TMPDIR: NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "akeru-computer-use-")),+ },+ } satisfies McpServerConfig;+}diff --git a/apps/server/src/provider/Layers/AgentController.test.ts b/apps/server/src/provider/Layers/AgentController.test.tsindex a8b6d2e5d..5fc2410ec 100644--- a/apps/server/src/provider/Layers/AgentController.test.ts+++ b/apps/server/src/provider/Layers/AgentController.test.ts@@ -71,6 +71,38 @@ const codexSelection = {instanceId: codexInstanceId,model: "gpt-5.6-sol",};+const computerUseToolName = "builtin-computer-use_control";++function computerUseServer() {+ return {+ id: McpServerId.make("builtin-computer-use"),+ name: "Codex Computer Use",+ transport: "stdio" as const,+ command: "akeru-codex-computer-use",+ enabled: true,+ createdAt: "2026-01-01T00:00:00.000Z",+ updatedAt: "2026-01-01T00:00:00.000Z",+ };+}++function computerUseMcpManager() {+ return {+ init: vi.fn(async () => undefined),+ disconnect: vi.fn(async () => undefined),+ getTools: vi.fn(() => ({+ [computerUseToolName]: { execute: vi.fn(async () => ({})) },+ })),+ getServerStatuses: vi.fn(() => [+ {+ name: "builtin-computer-use",+ connected: true,+ toolCount: 1,+ toolNames: [computerUseToolName],+ transport: "stdio" as const,+ },+ ]),+ };+}function makeProviderSession(threadId: ThreadId,@@ -259,12 +291,15 @@ function makeLayer(makeMcpManager?: NonNullable<AgentControllerLiveOptions["makeMcpManager"]>,baseDir?: string,usageLedger: BotUsageLedgerShape = makeUsageLedger().service,- memory?: Pick<AgentControllerLiveOptions, "entityMemoryRepository" | "memoryCandidateRepository">,+ overrides?: Pick<+ AgentControllerLiveOptions,+ "resolveComputerUseServer" | "entityMemoryRepository" | "memoryCandidateRepository"+ >,) {return makeAgentControllerLive({makeMastraHarness: factory,...(makeMcpManager ? { makeMcpManager } : {}),- ...memory,+ ...overrides,makeBotBrowser: () =>({tools: {},@@ -295,10 +330,13 @@ function provideController<A, E>(makeMcpManager?: NonNullable<AgentControllerLiveOptions["makeMcpManager"]>,baseDir?: string,usageLedger?: BotUsageLedgerShape,- memory?: Pick<AgentControllerLiveOptions, "entityMemoryRepository" | "memoryCandidateRepository">,+ overrides?: Pick<+ AgentControllerLiveOptions,+ "resolveComputerUseServer" | "entityMemoryRepository" | "memoryCandidateRepository"+ >,) {return effect.pipe(- Effect.provide(makeLayer(bridge, factory, makeMcpManager, baseDir, usageLedger, memory)),+ Effect.provide(makeLayer(bridge, factory, makeMcpManager, baseDir, usageLedger, overrides)),Effect.orDie,);}@@ -1672,6 +1710,189 @@ describe("AgentControllerLive", () => {);});+ it.effect("keeps Computer Use approval data and desktop content out of runtime events", () => {+ const bridge = makeBridge();+ const mastra = makeMastraHarness();+ const toolName = computerUseToolName;+ const mcpManager = computerUseMcpManager();+ const server = computerUseServer();++ return provideController(+ Effect.gen(function* () {+ const controller = yield* AgentController;+ yield* resolveCodex(controller);+ yield* controller.startSession(codexThreadId, {+ threadId: codexThreadId,+ provider: ProviderDriverKind.make("codex"),+ providerInstanceId: codexInstanceId,+ modelSelection: codexSelection,+ runtimeMode: "full-access",+ mcpServers: [server],+ });++ const events: ProviderRuntimeEvent[] = [];+ const eventsFiber = yield* controller.streamEvents.pipe(+ Stream.runForEach((event) => Effect.sync(() => events.push(event))),+ Effect.forkChild({ startImmediately: true }),+ );+ yield* Effect.yieldNow;+ yield* controller.sendTurn({ threadId: codexThreadId, input: "Control this Mac." });+ mastra.emit({+ type: "tool_start",+ toolCallId: "computer-use-1",+ toolName,+ args: { text: "typed secret", app: "Private App", path: "/private/tester" },+ } as AgentControllerEvent);+ mastra.emit({+ type: "tool_update",+ toolCallId: "computer-use-1",+ partialResult: { windowTitle: "Private Window" },+ } as AgentControllerEvent);+ mastra.emit({+ type: "tool_approval_required",+ toolCallId: "computer-use-1",+ toolName,+ args: { text: "typed secret", app: "Private App" },+ } as AgentControllerEvent);+ yield* controller.respondToRequest({+ threadId: codexThreadId,+ requestId: ApprovalRequestId.make("computer-use-1"),+ decision: "acceptForSession",+ });+ mastra.emit({+ type: "tool_end",+ toolCallId: "computer-use-1",+ result: { screenshot: "raw frame", applicationContent: "private content" },+ } as AgentControllerEvent);+ mastra.emit({+ type: "tool_approval_required",+ toolCallId: "computer-use-2",+ toolName,+ args: { text: "deny this" },+ } as AgentControllerEvent);+ yield* controller.respondToRequest({+ threadId: codexThreadId,+ requestId: ApprovalRequestId.make("computer-use-2"),+ decision: "decline",+ });+ mastra.emit({+ type: "error",+ error: new Error("Private Window at /private/tester failed"),+ } as AgentControllerEvent);+ yield* Effect.yieldNow;+ yield* Fiber.interrupt(eventsFiber);++ const serialized = JSON.stringify(events);+ expect(serialized).not.toContain("typed secret");+ expect(serialized).not.toContain("Private App");+ expect(serialized).not.toContain("Private Window");+ expect(serialized).not.toContain("private/tester");+ expect(serialized).not.toContain("raw frame");+ expect(serialized).not.toContain("private content");+ const approval = events.find(+ (event): event is Extract<ProviderRuntimeEvent, { readonly type: "request.opened" }> =>+ event.type === "request.opened" && event.requestId === "computer-use-1",+ );+ assert.isDefined(approval);+ assert.isDefined(approval.payload.options);+ expect(approval.payload.options.map((option) => option.decision)).toEqual([+ "accept",+ "decline",+ ]);+ expect(mastra.session.respondToToolApproval).toHaveBeenCalledWith({+ toolCallId: "computer-use-1",+ decision: "approve",+ });+ expect(mastra.session.respondToToolApproval).toHaveBeenCalledWith({+ toolCallId: "computer-use-2",+ decision: "decline",+ });+ expect(mastra.session.permissions.setForTool).not.toHaveBeenCalledWith({+ toolName,+ policy: "allow",+ });+ mastra.finishSend();+ }),+ bridge.service,+ mastra.factory,+ (() => mcpManager as never) as NonNullable<AgentControllerLiveOptions["makeMcpManager"]>,+ undefined,+ undefined,+ {+ resolveComputerUseServer: async () => ({+ command: "/local/launcher",+ args: ["mcp"],+ env: {},+ }),+ },+ );+ });++ it.effect("releases Computer Use when session setup and deletion fail", () => {+ const bridge = makeBridge();+ const mastra = makeMastraHarness();+ const mcpManager = computerUseMcpManager();+ const nextThreadId = ThreadId.make("thread-mastra-codex-next");+ vi.mocked(mastra.session.state.set).mockRejectedValueOnce(new Error("setup failed"));++ return provideController(+ Effect.gen(function* () {+ const controller = yield* AgentController;+ yield* resolveCodex(controller);+ yield* controller+ .startSession(codexThreadId, {+ threadId: codexThreadId,+ provider: ProviderDriverKind.make("codex"),+ providerInstanceId: codexInstanceId,+ modelSelection: codexSelection,+ runtimeMode: "full-access",+ mcpServers: [computerUseServer()],+ })+ .pipe(Effect.ignore);++ yield* controller.resolveEngine({+ threadId: nextThreadId,+ engine: { provider: "codex", model: "gpt-5.6-sol" },+ fallback: codexSelection,+ mode: "default",+ botConversation: true,+ });+ yield* controller.startSession(nextThreadId, {+ threadId: nextThreadId,+ provider: ProviderDriverKind.make("codex"),+ providerInstanceId: codexInstanceId,+ modelSelection: codexSelection,+ runtimeMode: "full-access",+ mcpServers: [computerUseServer()],+ });++ mastra.deleteSession.mockRejectedValueOnce(new Error("delete failed"));+ yield* controller.stopSession({ threadId: nextThreadId }).pipe(Effect.ignore);+ yield* controller.startSession(codexThreadId, {+ threadId: codexThreadId,+ provider: ProviderDriverKind.make("codex"),+ providerInstanceId: codexInstanceId,+ modelSelection: codexSelection,+ runtimeMode: "full-access",+ mcpServers: [computerUseServer()],+ });+ expect(mcpManager.init).toHaveBeenCalledTimes(3);+ }),+ bridge.service,+ mastra.factory,+ (() => mcpManager as never) as NonNullable<AgentControllerLiveOptions["makeMcpManager"]>,+ undefined,+ undefined,+ {+ resolveComputerUseServer: async () => ({+ command: "/local/launcher",+ args: ["mcp"],+ env: {},+ }),+ },+ );+ });+it.effect("creates a remote Mastra workspace for the bot sandbox provider", () => {const bridge = makeBridge();const mastra = makeMastraHarness();diff --git a/apps/server/src/provider/Layers/AgentController.ts b/apps/server/src/provider/Layers/AgentController.tsindex b5538fbb1..8f77d003b 100644--- a/apps/server/src/provider/Layers/AgentController.ts+++ b/apps/server/src/provider/Layers/AgentController.ts@@ -32,6 +32,7 @@ import {AKERU_PRODUCT_FEEDBACK_TOOL_NAME,type AkeruMemoryThreadAccess,} from "@t3tools/contracts";+import { HostProcessPlatform } from "@t3tools/shared/hostProcess";import * as Effect from "effect/Effect";import * as Layer from "effect/Layer";import * as PubSub from "effect/PubSub";@@ -83,6 +84,11 @@ import {} from "../AkeruToolRuntime.ts";import type { BotBrowser, BotBrowserAttachment, CreateBotBrowserInput } from "../botBrowser.ts";import { AkeruSessionResources } from "../AkeruSessionResources.ts";+import {+ isCodexComputerUseServer,+ isCodexComputerUseTool,+ resolveCodexComputerUseServer,+} from "../CodexComputerUse.ts";import type { AkeruBotWorkspace, CreateRemoteBotWorkspaceInput } from "../botWorkspace.ts";import {botRuntimeResourceScope,@@ -151,6 +157,7 @@ export interface AgentControllerLiveOptions {input: CreateRemoteBotWorkspaceInput,) => Promise<AkeruBotWorkspace | Workspace>;readonly makeBotBrowser?: (input: CreateBotBrowserInput) => BotBrowser;+ readonly resolveComputerUseServer?: typeof resolveCodexComputerUseServer;readonly entityMemoryRepository?: EntityMemoryRepositoryShape;readonly memoryCandidateRepository?: MemoryCandidateRepositoryShape;}@@ -163,6 +170,12 @@ function failureDetail(cause: unknown): string {return cause instanceof Error ? cause.message : String(cause);}+function sessionFailureDetail(active: Pick<ActiveSession, "mcpServerIds">, cause: unknown): string {+ return active.mcpServerIds.some((id) => isCodexComputerUseServer(String(id)))+ ? "Computer Use session failed."+ : failureDetail(cause);+}+function nowIso(): string {return new Date().toISOString();}@@ -319,6 +332,7 @@ function itemType(const make = (options?: AgentControllerLiveOptions) =>Effect.gen(function* () {const config = yield* ServerConfig;+ const hostPlatform = yield* HostProcessPlatform;const legacyProviderBridge = yield* LegacyProviderBridge;const botUsageLedger = yield* BotUsageLedger;const runtimeContext = yield* Effect.context<never>();@@ -362,12 +376,16 @@ const make = (options?: AgentControllerLiveOptions) =>const subscriptionAuth = SubscriptionAuthService.forSecretsDir(config.secretsDir);const sessionResources = new AkeruSessionResources({stateDir: config.stateDir,+ hostPlatform,toMcpServerConfigs,onMcpServerConnectionFailure: (serverId) =>subscriptionAuth.recordMcpRequestFailure(serverId, "The MCP server failed to connect."),...(options?.makeMcpManager ? { makeMcpManager: options.makeMcpManager } : {}),...(options?.makeRemoteWorkspace ? { makeRemoteWorkspace: options.makeRemoteWorkspace } : {}),...(options?.makeBotBrowser ? { makeBotBrowser: options.makeBotBrowser } : {}),+ ...(options?.resolveComputerUseServer+ ? { resolveComputerUseServer: options.resolveComputerUseServer }+ : {}),});const botInbox = BotInboxService.forSecretsDir(config.secretsDir);const toolRuntime = createAkeruToolRuntime({@@ -698,8 +716,10 @@ const make = (options?: AgentControllerLiveOptions) =>payload: {itemType: itemType(event.toolName),status: "inProgress",- title: event.toolName,- data: { args: event.args },+ title: isCodexComputerUseTool(event.toolName) ? "Computer Use" : event.toolName,+ data: isCodexComputerUseTool(event.toolName)+ ? { action: "computer-use" }+ : { args: event.args },},});return;@@ -713,7 +733,9 @@ const make = (options?: AgentControllerLiveOptions) =>payload: {itemType: itemType(active.toolNames.get(event.toolCallId) ?? "tool"),status: "inProgress",- data: { partialResult: event.partialResult },+ data: isCodexComputerUseTool(active.toolNames.get(event.toolCallId) ?? "")+ ? { action: "computer-use" }+ : { partialResult: event.partialResult },},});return;@@ -744,8 +766,10 @@ const make = (options?: AgentControllerLiveOptions) =>payload: {itemType: itemType(toolName),status: event.isError ? "failed" : event.denied ? "declined" : "completed",- title: toolName,- data: { result: event.result },+ title: isCodexComputerUseTool(toolName) ? "Computer Use" : toolName,+ data: isCodexComputerUseTool(toolName)+ ? { action: "computer-use" }+ : { result: event.result },},});return;@@ -766,14 +790,19 @@ const make = (options?: AgentControllerLiveOptions) =>type: "request.opened",payload: {requestType: "dynamic_tool_call",- detail:- event.toolName === AKERU_PRODUCT_FEEDBACK_TOOL_NAME+ detail: isCodexComputerUseTool(event.toolName)+ ? "Allow Computer Use?"+ : event.toolName === AKERU_PRODUCT_FEEDBACK_TOOL_NAME? "Review product feedback": `Allow ${event.toolName}?`,- toolName: event.toolName,- args: event.args,- options:- event.toolName === AKERU_PRODUCT_FEEDBACK_TOOL_NAME+ toolName: isCodexComputerUseTool(event.toolName) ? "Computer Use" : event.toolName,+ args: isCodexComputerUseTool(event.toolName) ? undefined : event.args,+ options: isCodexComputerUseTool(event.toolName)+ ? [+ { decision: "accept", label: "Allow" },+ { decision: "decline", label: "Decline" },+ ]+ : event.toolName === AKERU_PRODUCT_FEEDBACK_TOOL_NAME? [{ decision: "accept", label: "Add to feedback draft" },{ decision: "decline", label: "Cancel" },@@ -834,17 +863,19 @@ const make = (options?: AgentControllerLiveOptions) =>},});return;- case "error":+ case "error": {+ const detail = sessionFailureDetail(active, event.error);publish({...baseEvent(threadId, active, turn?.turnId),type: "runtime.error",payload: {- message: event.error.message || "Mastra agent failed.",+ message: detail,class: "provider_error",},});- finishTurn(threadId, active, "failed", event.error.message || "Mastra agent failed.");+ finishTurn(threadId, active, "failed", detail);return;+ }case "agent_end":if (event.reason === "suspended") {if (turn) turn.waiting = true;@@ -942,12 +973,30 @@ const make = (options?: AgentControllerLiveOptions) =>const workspaceId = botWorkspaceIdentity(workspaceResourceKey);const existing = sessions.get(key);const resolved = resolvedByThread.get(key);+ const mcpServers = input.mcpServers ?? [];+ if (!resolved) {+ return yield* new AgentControllerRuntimeError({+ operation: "startSession",+ detail: `Thread '${threadId}' has no resolved engine.`,+ });+ }+ if (+ mcpServers.some((server) => isCodexComputerUseServer(String(server.id))) &&+ resolved.provider !== ProviderDriverKind.make("codex")+ ) {+ return yield* new AgentControllerRuntimeError({+ operation: "startSession",+ detail: "Computer Use requires a Codex bot.",+ });+ }if (existing?.workspaceResourceKey === workspaceResourceKey &&existing.cwd === input.cwd &&resolved &&existing.provider === resolved.provider &&- existing.providerInstanceId === resolved.providerInstanceId+ existing.providerInstanceId === resolved.providerInstanceId &&+ existing.mcpServerIds.length === (input.mcpServers?.length ?? 0) &&+ existing.mcpServerIds.every((id) => input.mcpServers?.some((server) => server.id === id))) {existing.runtimeMode = input.runtimeMode;const toolSession = { ...existing.toolSession };@@ -991,13 +1040,6 @@ const make = (options?: AgentControllerLiveOptions) =>previousWorkspaceResourceKey !== workspaceResourceKey,);}- if (!resolved) {- return yield* new AgentControllerRuntimeError({- operation: "startSession",- detail: `Thread '${threadId}' has no resolved engine.`,- });- }- const mcpServers = input.mcpServers ?? [];const resources = yield* runMastra("resources.acquire", () =>sessionResources.acquire({threadId: key,@@ -1112,67 +1154,77 @@ const make = (options?: AgentControllerLiveOptions) =>),),);- yield* runMastra("state.set", () =>- session.state.set({- ...(input.cwd ? { projectPath: input.cwd } : {}),- yolo: false,- botConversation: resolved.botConversation,- }),- );- yield* runMastra("model.switch", () =>- session.model.switch({ modelId: resolved.mastraModelId }),- );- const modeId = mastraModeId(resolved.mode);- if (session.mode.get() !== modeId) {- yield* runMastra("mode.switch", () => session.mode.switch({ modeId }));- }- yield* Effect.forEach(- ["read", "edit", "execute", "mcp", "other"] as const,- (category) =>- runMastra("permissions.setForCategory", () =>- session.permissions.setForCategory({- category,- policy: permissionPolicy(input.runtimeMode, category),- }),- ),- { discard: true },- );- yield* Effect.forEach(- new Set([- ...toolRuntime.toolsForThread(key).map((tool) => tool.id),- ...Object.keys(sessionResources.getConnectorTools(key)),- AKERU_PRODUCT_FEEDBACK_TOOL_NAME,- "RestartMcpServers",- ]),- (toolName) =>- runMastra("permissions.setForTool", () =>- session.permissions.setForTool({ toolName, policy: "ask" }),- ),- { discard: true },- );- const createdAt = nowIso();- const activeWithoutUnsubscribe = {- session,- provider: resolved.provider,- providerInstanceId: resolved.providerInstanceId,- cwd: input.cwd,- createdAt,- mcpServerIds: mcpServers.map((server) => server.id),- runtimeMode: input.runtimeMode,- model: resolved.modelSelection.model,- status: "ready" as const,- activeTurn: null,- toolNames: new Map<string, string>(),- approvalRequests: new Map(),- connectorSessionApprovals: new Set<string>(),- toolSession,- workspaceResourceKey,- };- const unsubscribe = session.subscribe((event) => {- const active = sessions.get(key);- if (active) handleControllerEvent(threadId, active, event);+ const cleanupCreatedSession = Effect.gen(function* () {+ yield* runMastra("deleteSession", () =>+ bundle.controller.deleteSession({ resourceId: key }),+ ).pipe(Effect.ignoreCause({ log: true }));+ yield* runMastra("resources.release", () =>+ sessionResources.release(key, { destroy: true }),+ ).pipe(Effect.ignoreCause({ log: true }));+ toolRuntime.unregisterSession(key);});- const active: ActiveSession = { ...activeWithoutUnsubscribe, unsubscribe };+ const active = yield* Effect.gen(function* () {+ yield* runMastra("state.set", () =>+ session.state.set({+ ...(input.cwd ? { projectPath: input.cwd } : {}),+ yolo: false,+ botConversation: resolved.botConversation,+ }),+ );+ yield* runMastra("model.switch", () =>+ session.model.switch({ modelId: resolved.mastraModelId }),+ );+ const modeId = mastraModeId(resolved.mode);+ if (session.mode.get() !== modeId) {+ yield* runMastra("mode.switch", () => session.mode.switch({ modeId }));+ }+ yield* Effect.forEach(+ ["read", "edit", "execute", "mcp", "other"] as const,+ (category) =>+ runMastra("permissions.setForCategory", () =>+ session.permissions.setForCategory({+ category,+ policy: permissionPolicy(input.runtimeMode, category),+ }),+ ),+ { discard: true },+ );+ yield* Effect.forEach(+ new Set([+ ...toolRuntime.toolsForThread(key).map((tool) => tool.id),+ ...Object.keys(sessionResources.getConnectorTools(key)),+ AKERU_PRODUCT_FEEDBACK_TOOL_NAME,+ "RestartMcpServers",+ ]),+ (toolName) =>+ runMastra("permissions.setForTool", () =>+ session.permissions.setForTool({ toolName, policy: "ask" }),+ ),+ { discard: true },+ );+ const unsubscribe = session.subscribe((event) => {+ const current = sessions.get(key);+ if (current) handleControllerEvent(threadId, current, event);+ });+ return {+ session,+ provider: resolved.provider,+ providerInstanceId: resolved.providerInstanceId,+ cwd: input.cwd,+ createdAt: nowIso(),+ mcpServerIds: mcpServers.map((server) => server.id),+ runtimeMode: input.runtimeMode,+ model: resolved.modelSelection.model,+ status: "ready" as const,+ activeTurn: null,+ toolNames: new Map<string, string>(),+ approvalRequests: new Map(),+ connectorSessionApprovals: new Set<string>(),+ toolSession,+ workspaceResourceKey,+ unsubscribe,+ } satisfies ActiveSession;+ }).pipe(Effect.onError(() => cleanupCreatedSession));sessions.set(key, active);publish({...baseEvent(threadId, active),@@ -1281,7 +1333,7 @@ const make = (options?: AgentControllerLiveOptions) =>}}).catch((cause: unknown) => {- const detail = failureDetail(cause);+ const detail = sessionFailureDetail(active, cause);publish({...baseEvent(input.threadId, active, turnId),type: "runtime.error",@@ -1342,6 +1394,7 @@ const make = (options?: AgentControllerLiveOptions) =>if (input.decision === "acceptForSession" &&!runtimeToolId &&+ !isCodexComputerUseTool(toolName) &&!akeruActionNeedsApproval(toolName, toolInput) &&toolName !== AKERU_PRODUCT_FEEDBACK_TOOL_NAME) {@@ -1403,16 +1456,25 @@ const make = (options?: AgentControllerLiveOptions) =>if (!active) {const legacySessions = yield* legacyProviderBridge.listSessions();if (legacySessions.some((session) => session.threadId === input.threadId)) {- yield* legacyProviderBridge.stopSession(input);- yield* runMastra("resources.release", () =>- sessionResources.release(key, { destroy: destroyResources }),- ).pipe(Effect.ignoreCause({ log: true }));- legacyResourceIdentity.delete(key);+ yield* legacyProviderBridge.stopSession(input).pipe(+ Effect.ensuring(+ Effect.gen(function* () {+ yield* runMastra("resources.release", () =>+ sessionResources.release(key, { destroy: destroyResources }),+ ).pipe(Effect.ignoreCause({ log: true }));+ legacyResourceIdentity.delete(key);+ }),+ ),+ );return;}if (usesMastraCode(resolvedByThread.get(key)?.provider ?? ProviderDriverKind.make("codex"))) {+ yield* runMastra("resources.release", () =>+ sessionResources.release(key, { destroy: destroyResources }),+ ).pipe(Effect.ignoreCause({ log: true }));+ toolRuntime.unregisterSession(key);return;}return yield* legacyProviderBridge.stopSession(input);@@ -1421,13 +1483,20 @@ const make = (options?: AgentControllerLiveOptions) =>active.session.abort();active.unsubscribe();publishSessionState(input.threadId, active, "stopped");- yield* runMastra("deleteSession", () => bundle.controller.deleteSession({ resourceId: key }));- yield* runMastra("resources.release", () =>- sessionResources.release(key, { destroy: destroyResources }),- ).pipe(Effect.ignoreCause({ log: true }));- toolRuntime.unregisterSession(key);- sessions.delete(key);- memoryUsageByThread.delete(key);+ yield* runMastra("deleteSession", () =>+ bundle.controller.deleteSession({ resourceId: key }),+ ).pipe(+ Effect.ensuring(+ Effect.gen(function* () {+ yield* runMastra("resources.release", () =>+ sessionResources.release(key, { destroy: destroyResources }),+ ).pipe(Effect.ignoreCause({ log: true }));+ toolRuntime.unregisterSession(key);+ sessions.delete(key);+ memoryUsageByThread.delete(key);+ }),+ ),+ );});const stopSession: AgentControllerShape["stopSession"] = (input) =>diff --git a/apps/web/src/components/plugins/pluginPresentation.test.ts b/apps/web/src/components/plugins/pluginPresentation.test.tsindex 121cefb21..8cafa1fab 100644--- a/apps/web/src/components/plugins/pluginPresentation.test.ts+++ b/apps/web/src/components/plugins/pluginPresentation.test.ts@@ -92,7 +92,7 @@ describe("plugin presentation", () => {buildPluginSections({ plugins: catalog, query: "", filter: "All" }).flatMap((section) =>section.plugins.map((plugin) => plugin.id),),- ).toHaveLength(51);+ ).toHaveLength(52);expect(buildPluginSections({plugins: catalog,diff --git a/apps/web/src/components/sidebar/SidebarChrome.test.ts b/apps/web/src/components/sidebar/SidebarChrome.test.tsindex e7b682602..dbdcad7ef 100644--- a/apps/web/src/components/sidebar/SidebarChrome.test.ts+++ b/apps/web/src/components/sidebar/SidebarChrome.test.ts@@ -1,11 +1,15 @@// @effect-diagnostics nodeBuiltinImport:off - This integration guard reads related source files.import * as NodeFS from "node:fs";-import { McpServerId, type McpServer } from "@t3tools/contracts";+import { BotId, McpServerId, ThreadId, type McpServer } from "@t3tools/contracts";import { describe, expect, it } from "vite-plus/test";import { loadCatalog } from "../../../../../plugins";-import { formatEnabledPluginStatus, summarizeEnabledPlugins } from "./SidebarChrome";+import {+ findActiveComputerUseControl,+ formatEnabledPluginStatus,+ summarizeEnabledPlugins,+} from "./SidebarChrome";const catalogPlugin = loadCatalog().find((plugin) => plugin.id === "exa");if (!catalogPlugin) throw new TypeError("Exa must remain in the plugin catalog.");@@ -39,6 +43,95 @@ describe("sidebar footer", () => {expect(summary.enabledPlugins).toEqual([catalogPlugin]);});+ it("shows only the bot that holds an enabled Computer Use session", () => {+ const botId = BotId.make("bot-1");+ const input: Parameters<typeof findActiveComputerUseControl>[0] = {+ mcpServers: [server("builtin-computer-use")],+ bots: [+ {+ id: botId,+ name: "Operator",+ title: "Operator",+ label: null,+ description: "",+ engine: null,+ sandbox: null,+ runtimeMode: "approval-required",+ usageCap: null,+ voiceEnabled: false,+ groupId: null,+ disabledMcpServerIds: [],+ avatar: { kind: "blob", shape: "circle", color: "#5B7FD4" },+ archivedAt: null,+ createdAt: "2026-08-27T00:00:00.000Z",+ updatedAt: "2026-08-27T00:00:00.000Z",+ },+ ],+ threads: [+ {+ id: ThreadId.make("thread-1"),+ botId,+ projectId: "project-1" as never,+ title: "Control",+ modelSelection: { instanceId: "codex" as never, model: "gpt-5.6-sol" },+ runtimeMode: "approval-required",+ interactionMode: "default",+ branch: null,+ worktreePath: null,+ latestTurn: null,+ createdAt: "2026-08-27T00:00:00.000Z",+ updatedAt: "2026-08-27T00:00:00.000Z",+ archivedAt: null,+ settledOverride: null,+ settledAt: null,+ session: {+ threadId: ThreadId.make("thread-1"),+ status: "ready",+ providerName: "codex",+ runtimeMode: "approval-required",+ mcpServerIds: [McpServerId.make("builtin-computer-use")],+ activeTurnId: null,+ lastError: null,+ updatedAt: "2026-08-27T00:00:00.000Z",+ },+ latestUserMessageAt: null,+ hasPendingApprovals: false,+ hasPendingUserInput: false,+ hasActionableProposedPlan: false,+ },+ ],+ };+ const control = findActiveComputerUseControl(input);++ expect(control).toEqual({ threadId: ThreadId.make("thread-1"), botName: "Operator" });+ for (const status of ["starting", "error", "stopped"] as const) {+ expect(+ findActiveComputerUseControl({+ ...input,+ threads: input.threads.map((thread) => ({+ ...thread,+ session: thread.session ? { ...thread.session, status } : null,+ })),+ }),+ ).toBeNull();+ }+ expect(+ findActiveComputerUseControl({+ ...input,+ mcpServers: [server("builtin-computer-use", false)],+ }),+ ).toBeNull();+ expect(+ findActiveComputerUseControl({+ ...input,+ threads: input.threads.map((thread) => ({+ ...thread,+ session: thread.session ? { ...thread.session, mcpServerIds: [] } : null,+ })),+ }),+ ).toBeNull();+ });+it("keeps the error inbox in Settings instead of the sidebar", () => {const source = NodeFS.readFileSync(new URL("./SidebarChrome.tsx", import.meta.url), "utf8");diff --git a/apps/web/src/components/sidebar/SidebarChrome.tsx b/apps/web/src/components/sidebar/SidebarChrome.tsxindex 3eb88668b..bc3b404dd 100644--- a/apps/web/src/components/sidebar/SidebarChrome.tsx+++ b/apps/web/src/components/sidebar/SidebarChrome.tsx@@ -5,10 +5,16 @@ import {PlugSocketIcon,Settings02Icon,} from "@hugeicons/core-free-icons";-import type { EnvironmentId, McpServer } from "@t3tools/contracts";+import type {+ EnvironmentId,+ McpServer,+ OrchestrationBot,+ OrchestrationThreadShell,+ ThreadId,+} from "@t3tools/contracts";import { Link } from "@tanstack/react-router";import type { ReactNode } from "react";-import { memo, useCallback } from "react";+import { memo, useCallback, useState } from "react";import {loadCatalog,resolveCatalogInstallations,@@ -21,7 +27,10 @@ import { openPlugins } from "../../pluginsDialogStore";import { openUsage } from "../../usageDialogStore";import { openProductFeedback } from "../../productFeedbackStore";import { usePrimaryEnvironmentId } from "../../state/environments";-import { environmentMcpServersAtom } from "../../state/mcpServers";+import { environmentMcpServersAtom, mcpServerEnvironment } from "../../state/mcpServers";+import { environmentSnapshotAtom } from "../../state/shell";+import { threadEnvironment } from "../../state/threads";+import { useAtomCommand } from "../../state/use-atom-command";import { AkeruMark } from "../AkeruMark";import { T3ConnectSidebarAvatar, T3ConnectSidebarSignIn } from "../clerk/T3ConnectSidebarSignIn";import { PluginLogoImage } from "../plugins/PluginsCatalog";@@ -35,6 +44,7 @@ import {useEnvironmentStageLabel,} from "../SidebarStageBackdrop";import { Badge } from "../ui/badge";+import { Button } from "../ui/button";import { AppIcon } from "../ui/app-icon";import {SidebarFooter,@@ -120,6 +130,38 @@ function SidebarBrand({ onBackdrop }: { onBackdrop: boolean }) {}const PLUGIN_CATALOG = loadCatalog();+const COMPUTER_USE_SERVER_ID = "builtin-computer-use";++export interface ActiveComputerUseControl {+ readonly threadId: ThreadId;+ readonly botName: string;+}++export function findActiveComputerUseControl(input: {+ readonly threads: readonly OrchestrationThreadShell[];+ readonly bots: readonly OrchestrationBot[];+ readonly mcpServers: readonly McpServer[];+}): ActiveComputerUseControl | null {+ const server = input.mcpServers.find(+ (candidate) => candidate.id === COMPUTER_USE_SERVER_ID && candidate.enabled,+ );+ if (!server) return null;+ for (const thread of input.threads) {+ if (+ !thread.session ||+ thread.session.providerName?.toLowerCase() !== "codex" ||+ !thread.session.mcpServerIds?.includes(server.id) ||+ (thread.session.status !== "ready" && thread.session.status !== "running")+ ) {+ continue;+ }+ const botId = thread.respondingBotId ?? thread.botId;+ const bot = input.bots.find((candidate) => candidate.id === botId);+ if (!bot || bot.disabledMcpServerIds.includes(server.id)) continue;+ return { threadId: thread.id, botName: bot.name };+ }+ return null;+}export function formatEnabledPluginStatus(enabledCount: number): string {if (enabledCount === 0) return "No plugins enabled";@@ -224,6 +266,81 @@ function SidebarPluginSummary({ onClick }: { readonly onClick: () => void }) {return <SidebarPluginSummaryForEnvironment environmentId={environmentId} onClick={onClick} />;}+function ComputerUseControlForEnvironment({+ environmentId,+}: {+ readonly environmentId: EnvironmentId;+}) {+ const snapshot = useAtomValue(environmentSnapshotAtom(environmentId));+ const stopSession = useAtomCommand(threadEnvironment.stopSession);+ const disableServer = useAtomCommand(mcpServerEnvironment.disable);+ const [pending, setPending] = useState(false);+ const control = snapshot+ ? findActiveComputerUseControl({+ threads: snapshot.threads,+ bots: snapshot.bots,+ mcpServers: snapshot.mcpServers ?? [],+ })+ : null;+ if (!control) return null;++ const stop = async () => {+ setPending(true);+ try {+ await stopSession({ environmentId, input: { threadId: control.threadId } });+ } finally {+ setPending(false);+ }+ };+ const revoke = async () => {+ setPending(true);+ try {+ const stopped = await stopSession({+ environmentId,+ input: { threadId: control.threadId },+ });+ if (stopped._tag === "Success") {+ await disableServer({+ environmentId,+ input: { mcpServerId: COMPUTER_USE_SERVER_ID as McpServer["id"] },+ });+ }+ } finally {+ setPending(false);+ }+ };++ return (+ <div className="rounded-xl border border-amber-500/30 bg-amber-500/10 p-2.5" role="status">+ <div className="flex items-center gap-2 text-xs font-medium">+ <span className="size-2 rounded-full bg-amber-500" aria-hidden="true" />+ <span className="min-w-0 truncate">{control.botName} controls this Mac</span>+ </div>+ <div className="mt-2 flex gap-2">+ <Button className="h-7 flex-1 text-xs" disabled={pending} size="sm" onClick={stop}>+ Stop+ </Button>+ <Button+ aria-label="Revoke Computer Use for all bots"+ className="h-7 flex-1 text-xs"+ disabled={pending}+ size="sm"+ title="Disable Computer Use for all bots"+ variant="destructive-outline"+ onClick={revoke}+ >+ Revoke+ </Button>+ </div>+ </div>+ );+}++function ComputerUseControl() {+ const environmentId = usePrimaryEnvironmentId();+ return environmentId ? <ComputerUseControlForEnvironment environmentId={environmentId} /> : null;+}+function SidebarUtilityItem({icon,label,@@ -279,6 +396,7 @@ export const SidebarChromeFooter = memo(function SidebarChromeFooter() {return (<SidebarFooter className="max-h-[min(45dvh,22rem)] shrink-0 overflow-y-auto overscroll-contain p-[var(--sidebar-content-inset)]"><div className="flex flex-col gap-2 empty:hidden group-data-[collapsible=icon]:hidden">+ <ComputerUseControl /><SidebarProviderUpdatePill /><SidebarUpdateArchitectureWarning /></div>diff --git a/docs/internals/plugin-lifecycle-verification.md b/docs/internals/plugin-lifecycle-verification.mdindex 9a8e0207e..e80d280bf 100644--- a/docs/internals/plugin-lifecycle-verification.md+++ b/docs/internals/plugin-lifecycle-verification.md@@ -2,7 +2,7 @@Milestone 13 keeps catalog discovery separate from connection verification. An entry stays non-installable until its real install, authentication, safe read, approved write, disable, reconnect, and removal lifecycle passes. `verification-pending` means that local lifecycle proof is missing. `approval-pending` means an external vendor, administrator, allowlist, or first-party connector is still required.-`plugins/lifecycle-matrix.test.ts` is the catalog gate. It fixes the directory at 51 IDs, keeps four verified `builtin-<id>` recipes installable, and keeps the other 47 entries pending. It also checks Featured order, consequential approval coverage, broker identity, Custom MCP independence, legacy built-in display, and the Executor, Typefully, Paper, and PayPal blockers.+`plugins/lifecycle-matrix.test.ts` is the catalog gate. It fixes the directory at 52 IDs, keeps four verified `builtin-<id>` recipes installable, and keeps the other 48 entries pending. It also checks Featured order, consequential approval coverage, broker identity, Custom MCP independence, legacy built-in display, and the Computer Use, Executor, Typefully, Paper, and PayPal blockers.## Runtime proofdiff --git a/docs/user/plugins.md b/docs/user/plugins.mdindex d5b429c88..33899fb12 100644--- a/docs/user/plugins.md+++ b/docs/user/plugins.md@@ -15,6 +15,8 @@ The available action matches the current state:Approval-pending plugins remain visible, but Akeru disables **Connect** and shows the vendor blocker. Plugins removed from the directory remain under **Installed** until you remove them.+Codex Computer Use runs only on the local Mac. It does not use a hosted desktop or remote-control service. When a Codex bot starts control, macOS requests Screen Recording and Accessibility access if the helper does not have them. Akeru allows one controlling bot at a time and shows its name in the sidebar. **Stop** ends that session. **Revoke** ends it and disables Computer Use for every bot. Akeru keeps screenshots, typed text, window titles, and application content out of stored runtime events.+Akeru enables each installed plugin for every bot by default. Use a bot's **Plugins** setting to disable it for that bot. A provider change starts a fresh provider session with the same enabled plugins.Ask the agent to add a Custom MCP server when a connector is not in the directory. Existing Custom MCP servers remain under **Installed**, where you can edit, disable, or delete them. Akeru stores no plugin credentials in the directory or MCP registry. Keep credentials in the host environment or the service sign-in flow.diff --git a/packages/contracts/src/orchestration.ts b/packages/contracts/src/orchestration.tsindex 1731a003c..e12a3a9aa 100644--- a/packages/contracts/src/orchestration.ts+++ b/packages/contracts/src/orchestration.ts@@ -414,6 +414,7 @@ export const OrchestrationSession = Schema.Struct({providerName: Schema.NullOr(TrimmedNonEmptyString),providerInstanceId: Schema.optional(ProviderInstanceId),runtimeMode: RuntimeMode.pipe(Schema.withDecodingDefault(Effect.succeed(DEFAULT_RUNTIME_MODE))),+ mcpServerIds: Schema.optional(Schema.Array(McpServerId)),activeTurnId: Schema.NullOr(TurnId),lastError: Schema.NullOr(TrimmedNonEmptyString),updatedAt: IsoDateTime,diff --git a/plugins/catalog.test.ts b/plugins/catalog.test.tsindex a665d6215..d96477572 100644--- a/plugins/catalog.test.ts+++ b/plugins/catalog.test.ts@@ -20,6 +20,7 @@ const EXPECTED_DIRECTORY_IDS = ["canva","cloudflare","coda",+ "computer-use","context","customer-io","datadog",@@ -165,7 +166,7 @@ describe("plugin catalog loader", () => {const directory = loadDirectoryCatalog();const catalog = loadCatalog();expect(directory.map((plugin) => plugin.id).toSorted()).toEqual(EXPECTED_DIRECTORY_IDS);- expect(new Set(directory.map((plugin) => plugin.id)).size).toBe(51);+ expect(new Set(directory.map((plugin) => plugin.id)).size).toBe(52);expect(catalog.map((plugin) => plugin.id)).toEqual(EXPECTED_INSTALLABLE_IDS);expect(catalog.map((plugin) => `builtin-${plugin.id}`)).toEqual(EXPECTED_INSTALLABLE_IDS.map((id) => `builtin-${id}`),@@ -217,13 +218,13 @@ describe("plugin catalog loader", () => {const pending = loadDirectoryCatalog().filter((plugin) => !EXPECTED_INSTALLABLE_IDS.some((id) => id === plugin.id),);- expect(pending).toHaveLength(47);+ expect(pending).toHaveLength(48);expect(pending.filter((plugin) => plugin.catalogStatus === "approval-pending")).toHaveLength(16,);expect(pending.filter((plugin) => plugin.catalogStatus === "verification-pending"),- ).toHaveLength(31);+ ).toHaveLength(32);for (const plugin of pending) {expect(["approval-pending", "verification-pending"]).toContain(plugin.catalogStatus);expect(plugin.connection).toMatchObject({diff --git a/plugins/entries/computer-use/logo-dark.svg b/plugins/entries/computer-use/logo-dark.svgnew file mode 100644index 000000000..48517141a--- /dev/null+++ b/plugins/entries/computer-use/logo-dark.svg@@ -0,0 +1,5 @@+<svg width="24" height="24" viewBox="0 0 40 40" xmlns="http://www.w3.org/2000/svg">+ <circle cx="20" cy="20" r="14" fill="none" stroke="currentColor" stroke-width="3"/>+ <path d="M20 7v26M8.7 13.5l22.6 13M8.7 26.5l22.6-13" fill="none" stroke="currentColor" stroke-width="3" stroke-linecap="round"/>+ <circle cx="20" cy="20" r="5" fill="currentColor"/>+</svg>diff --git a/plugins/entries/computer-use/logo.svg b/plugins/entries/computer-use/logo.svgnew file mode 100644index 000000000..48517141a--- /dev/null+++ b/plugins/entries/computer-use/logo.svg@@ -0,0 +1,5 @@+<svg width="24" height="24" viewBox="0 0 40 40" xmlns="http://www.w3.org/2000/svg">+ <circle cx="20" cy="20" r="14" fill="none" stroke="currentColor" stroke-width="3"/>+ <path d="M20 7v26M8.7 13.5l22.6 13M8.7 26.5l22.6-13" fill="none" stroke="currentColor" stroke-width="3" stroke-linecap="round"/>+ <circle cx="20" cy="20" r="5" fill="currentColor"/>+</svg>diff --git a/plugins/entries/computer-use/plugin.json b/plugins/entries/computer-use/plugin.jsonnew file mode 100644index 000000000..5bd3975d4--- /dev/null+++ b/plugins/entries/computer-use/plugin.json@@ -0,0 +1,56 @@+{+ "schemaVersion": 1,+ "id": "computer-use",+ "name": "Codex Computer Use",+ "description": "Let one Codex bot inspect and control local macOS applications with approval for every action.",+ "primaryCategory": "Build",+ "tags": ["codex", "computer-use", "macos", "desktop", "local"],+ "capabilities": [+ "capture the screen",+ "inspect local applications",+ "click and scroll",+ "type text and keyboard shortcuts",+ "open local applications"+ ],+ "publisher": { "name": "OpenAI", "url": "https://openai.com" },+ "maintainer": { "name": "OpenAI", "url": "https://openai.com" },+ "documentationUrl": "https://developers.openai.com/codex/computer-use",+ "sourceUrl": "https://developers.openai.com/codex/plugins",+ "license": "Proprietary",+ "logo": {+ "provenance": {+ "sourceUrl": "https://openai.com/brand/",+ "license": "OpenAI brand asset"+ }+ },+ "platforms": ["macos"],+ "transport": {+ "type": "stdio",+ "command": "akeru-codex-computer-use",+ "args": ["mcp"]+ },+ "connection": {+ "type": "verification-pending",+ "blocker": "A live Akeru Codex tool call still needs verification with a signed-in disposable environment."+ },+ "authentication": "none",+ "requiredCredentials": [],+ "setup": [+ "Install the official plugin with codex plugin add computer-use@openai-bundled --json.",+ "Use a Codex bot on macOS. Screen Recording and Accessibility are requested only when control starts."+ ],+ "permissions": [+ {+ "id": "inspect-local-desktop",+ "description": "Capture the screen and inspect local applications in memory.",+ "approval": "read"+ },+ {+ "id": "control-local-desktop",+ "description": "Click, scroll, type, use keyboard shortcuts, and open local applications.",+ "approval": "production"+ }+ ],+ "approvals": ["production"],+ "catalogStatus": "verification-pending"+}diff --git a/plugins/lifecycle-matrix.test.ts b/plugins/lifecycle-matrix.test.tsindex dc55d21be..d3be6926a 100644--- a/plugins/lifecycle-matrix.test.ts+++ b/plugins/lifecycle-matrix.test.ts@@ -9,7 +9,7 @@ import {} from "./catalog";const EXPECTED_IDS =- "ahrefs apify apollo asana atlassian attio canva cloudflare coda context customer-io datadog docusign dropbox exa executor figma firecrawl framer github help-scout hubspot intercom lemon-squeezy linear mobbin monday netlify notion paddle paper parallel-search paypal pipedrive posthog railway render salesforce semrush sentry sequenzy shopify slack stripe superside tavily typefully vercel webflow zendesk zernio".split(+ "ahrefs apify apollo asana atlassian attio canva cloudflare coda computer-use context customer-io datadog docusign dropbox exa executor figma firecrawl framer github help-scout hubspot intercom lemon-squeezy linear mobbin monday netlify notion paddle paper parallel-search paypal pipedrive posthog railway render salesforce semrush sentry sequenzy shopify slack stripe superside tavily typefully vercel webflow zendesk zernio".split(" ",);const INSTALLABLE_IDS = ["context", "exa", "firecrawl", "parallel-search"];@@ -35,13 +35,13 @@ describe("milestone 13 plugin lifecycle matrix", () => {]);const pending = directory.filter((plugin) => !INSTALLABLE_IDS.includes(plugin.id));- expect(pending).toHaveLength(47);+ expect(pending).toHaveLength(48);expect(pending.filter((plugin) => plugin.catalogStatus === "approval-pending")).toHaveLength(16,);expect(pending.filter((plugin) => plugin.catalogStatus === "verification-pending"),- ).toHaveLength(31);+ ).toHaveLength(32);for (const plugin of pending) {expect(["approval-pending", "verification-pending"]).toContain(plugin.catalogStatus);expect(plugin.connection).toMatchObject({@@ -66,6 +66,13 @@ describe("milestone 13 plugin lifecycle matrix", () => {transport: { type: "url", url: "https://api.githubcopilot.com/mcp/" },connection: { type: "verification-pending" },});+ expect(byId.get("computer-use")).toMatchObject({+ platforms: ["macos"],+ connection: {+ type: "verification-pending",+ blocker: expect.stringContaining("live Akeru Codex tool call"),+ },+ });expect(byId.get("paper")).toMatchObject({transport: { type: "url", url: "http://127.0.0.1:29979/mcp" },connection: { type: "verification-pending" },diff --git a/plugins/schema.ts b/plugins/schema.tsindex 26ac8579a..c924b8e2f 100644--- a/plugins/schema.ts+++ b/plugins/schema.ts@@ -393,7 +393,11 @@ function validateManifest(manifest: PluginManifest): PluginManifest {`Plugin '${manifest.id}' labels an API key connection without API key authentication.`,);}- if (manifest.transport.type === "stdio" && manifest.connection.type !== "local") {+ if (+ manifest.transport.type === "stdio" &&+ manifest.connection.type !== "local" &&+ manifest.connection.type !== "verification-pending"+ ) {throw new TypeError(`Plugin '${manifest.id}' must label its stdio connection as local.`);}if (diff --git a/scripts/validate-plugin-catalog.test.ts b/scripts/validate-plugin-catalog.test.tsindex 4a8d19401..7e64f1893 100644--- a/scripts/validate-plugin-catalog.test.ts+++ b/scripts/validate-plugin-catalog.test.ts@@ -29,7 +29,7 @@ afterEach(() => {describe("plugin catalog validation", () => {it("validates the live plugin directory", () => {- expect(validatePluginCatalog()).toHaveLength(51);+ expect(validatePluginCatalog()).toHaveLength(52);});it("validates isolated manifests and official local assets", () => {