Skip to content

Open-source Grok Bot alternative for named AI teammates. Local-first desktop, web, and mobile clients with per-bot tools, instructions, and memory. Independent fork of T3 Code.

fix(server): redact screenshot tool results (#62)

Leo365b017d8093
diff --git a/apps/server/src/provider/AkeruToolRuntime.test.ts b/apps/server/src/provider/AkeruToolRuntime.test.ts
index 2adbe69c5..2ae38e375 100644
--- a/apps/server/src/provider/AkeruToolRuntime.test.ts
+++ b/apps/server/src/provider/AkeruToolRuntime.test.ts
@@ -3,12 +3,23 @@ import * as NodeFS from "node:fs";
import * as NodeOS from "node:os";
import * as NodePath from "node:path";
-import { LocalFilesystem, LocalSandbox, Workspace } from "@mastra/core/workspace";
+import {
+ createWorkspaceTools,
+ LocalFilesystem,
+ LocalSandbox,
+ Workspace,
+} from "@mastra/core/workspace";
+import { PNG } from "pngjs";
import { afterEach, describe, expect, it, vi } from "vite-plus/test";
import { BotId } from "@t3tools/contracts";
import { createAkeruToolRuntime } from "./AkeruToolRuntime.ts";
+vi.mock("@mastra/core/workspace", async (importOriginal) => {
+ const actual = await importOriginal<typeof import("@mastra/core/workspace")>();
+ return { ...actual, createWorkspaceTools: vi.fn(actual.createWorkspaceTools) };
+});
+
const directories = new Set<string>();
function workspace(name: string) {
@@ -42,6 +53,41 @@ describe("AkeruToolRuntime", () => {
]);
});
+ it("redacts screenshot frames before returning tool results", async () => {
+ const png = new PNG({ width: 1, height: 1 });
+ png.data.set([255, 255, 255, 255]);
+ const bot = workspace("screenshot");
+ Object.defineProperty(bot.sandbox, "computer", { value: {} });
+ vi.mocked(createWorkspaceTools).mockResolvedValueOnce({
+ mastra_workspace_computer_screenshot: {
+ execute: async () => ({
+ __workspaceMedia: true,
+ text: "Screenshot captured.",
+ mediaType: "image/png",
+ data: PNG.sync.write(png).toString("base64"),
+ }),
+ },
+ });
+ const runtime = createAkeruToolRuntime();
+ runtime.registerSession("thread-screenshot", {
+ runtimeMode: "full-access",
+ workspaceType: "cloud",
+ workspace: bot,
+ });
+
+ const result = (await runtime.execute({
+ threadId: "thread-screenshot",
+ toolId: "Screenshot",
+ toolCallId: "tool-screenshot",
+ input: {},
+ approvalMode: "require-grant",
+ })) as { readonly data: string; readonly text: string };
+ const frame = PNG.sync.read(Buffer.from(result.data, "base64"));
+
+ expect(result.text).toBe("Screenshot captured.");
+ expect([...frame.data]).toEqual([0, 0, 0, 255]);
+ });
+
it("exposes registered memory handlers and protects sensitive writes", async () => {
const remember = vi.fn(async () => ({ saved: true }));
const unavailable = vi.fn(async () => undefined);
diff --git a/apps/server/src/provider/AkeruToolRuntime.ts b/apps/server/src/provider/AkeruToolRuntime.ts
index e139e2c2b..f300b95e9 100644
--- a/apps/server/src/provider/AkeruToolRuntime.ts
+++ b/apps/server/src/provider/AkeruToolRuntime.ts
@@ -14,6 +14,7 @@ import {
import * as Schema from "effect/Schema";
import type { UserActionIncidentInput } from "../bot-inbox/userActionIncidents.ts";
+import { redactComputerScreenshot } from "../mcp/PreviewSnapshotRedaction.ts";
import {
AkeruMemoryToolInputSchemas,
type AkeruMemoryToolHandler,
@@ -319,7 +320,7 @@ export function createAkeruToolRuntime(options?: AkeruToolRuntimeOptions): Akeru
input.toolId === "AwaitShell" || input.toolId === "AwaitExternalShell"
? { pid: requiredString(decoded, "handleId"), wait: true }
: decoded;
- return backend.execute(backendInput, {
+ const result = await backend.execute(backendInput, {
workspace,
requestContext: new RequestContext(),
observe: {
@@ -327,6 +328,21 @@ export function createAkeruToolRuntime(options?: AkeruToolRuntimeOptions): Akeru
log: () => undefined,
},
});
+ if (input.toolId !== "Screenshot") return result;
+
+ const mediaType = field(result, "mediaType");
+ const data = field(result, "data");
+ if (mediaType !== "image/png" || typeof data !== "string") {
+ throw new Error("Screenshot result is invalid.");
+ }
+ const redacted = redactComputerScreenshot({
+ mediaType,
+ data: Buffer.from(data, "base64"),
+ });
+ return {
+ ...(result as Record<string, unknown>),
+ data: Buffer.from(redacted.data).toString("base64"),
+ };
},
};
}