Skip to content

Open-source Grok Bot alternative for named AI teammates. Local-first desktop, web, and mobile clients with per-bot tools, instructions, and memory. Independent fork of T3 Code.

fix(server): redact browser snapshot data (#25)

Leo894766ed7b78
diff --git a/apps/server/package.json b/apps/server/package.json
index 31f7f8c0c..e0e240169 100644
--- a/apps/server/package.json
+++ b/apps/server/package.json
@@ -39,6 +39,7 @@
"effect": "catalog:",
"msgpackr-extract": "3.0.4",
"node-pty": "^1.1.0",
+ "pngjs": "7.0.0",
"yaml": "catalog:"
},
"devDependencies": {
@@ -49,6 +50,7 @@
"@t3tools/web": "workspace:*",
"@types/bun": "1.3.14",
"@types/node": "catalog:",
+ "@types/pngjs": "6.0.5",
"effect-acp": "workspace:*",
"effect-codex-app-server": "workspace:*",
"vite-plus": "catalog:"
diff --git a/apps/server/src/mcp/McpHttpServer.test.ts b/apps/server/src/mcp/McpHttpServer.test.ts
index fa2880f9c..d97fd2901 100644
--- a/apps/server/src/mcp/McpHttpServer.test.ts
+++ b/apps/server/src/mcp/McpHttpServer.test.ts
@@ -7,6 +7,7 @@ import * as Layer from "effect/Layer";
import * as Stream from "effect/Stream";
import { McpProtocol, McpSchema, McpServer } from "effect/unstable/ai";
import { HttpBody, HttpClient, HttpRouter, HttpServerResponse } from "effect/unstable/http";
+import { PNG } from "pngjs";
import * as McpHttpServer from "./McpHttpServer.ts";
import * as McpInvocationContext from "./McpInvocationContext.ts";
@@ -16,6 +17,11 @@ const environmentId = EnvironmentId.make("environment-mcp-test");
const threadId = ThreadId.make("thread-mcp-test");
const tabId = PreviewTabId.make("tab-mcp-test");
const alternateTabId = PreviewTabId.make("tab-mcp-alternate");
+const screenshot = (() => {
+ const png = new PNG({ width: 10, height: 5 });
+ png.data.fill(255);
+ return PNG.sync.write(png).toString("base64");
+})();
const invocation = {
environmentId,
threadId,
@@ -178,7 +184,7 @@ it.effect("registers annotated tools and preserves authenticated request context
url: "http://example.test/",
title: "Example",
loading: false,
- visibleText: "Example",
+ visibleText: "[email protected] token=secret-value",
interactiveElements: [],
accessibilityTree: {},
consoleEntries: [],
@@ -186,7 +192,7 @@ it.effect("registers annotated tools and preserves authenticated request context
actionTimeline: [],
screenshot: {
mimeType: "image/png",
- data: Buffer.from("png").toString("base64"),
+ data: screenshot,
width: 10,
height: 5,
},
@@ -259,8 +265,13 @@ it.effect("registers annotated tools and preserves authenticated request context
expect(snapshot.isError).toBe(false);
expect(snapshot.content.some((content) => content.type === "image")).toBe(true);
expect(snapshot.structuredContent).toMatchObject({
- screenshot: { mimeType: "image/png", width: 10, height: 5 },
+ visibleText: "[REDACTED] [REDACTED]",
+ screenshot: { mimeType: "image/png", width: 10, height: 5, redacted: true },
});
+ const image = snapshot.content.find((content) => content.type === "image");
+ if (!image || image.type !== "image") throw new Error("Expected a redacted snapshot image.");
+ const png = PNG.sync.read(Buffer.from(image.data));
+ expect([...png.data.subarray(0, 4)]).toEqual([0, 0, 0, 255]);
expect(routedRequests.find(({ operation }) => operation === "snapshot")?.tabId).toBe(
alternateTabId,
);
diff --git a/apps/server/src/mcp/McpHttpServer.ts b/apps/server/src/mcp/McpHttpServer.ts
index 87975a49d..dfcb5ac24 100644
--- a/apps/server/src/mcp/McpHttpServer.ts
+++ b/apps/server/src/mcp/McpHttpServer.ts
@@ -10,6 +10,7 @@ import { McpProtocol, McpSchema, McpServer, Tool } from "effect/unstable/ai";
import { HttpRouter, HttpServerRequest, HttpServerResponse } from "effect/unstable/http";
import packageJson from "../../package.json" with { type: "json" };
+import { redactPreviewSnapshot } from "./PreviewSnapshotRedaction.ts";
import * as McpInvocationContext from "./McpInvocationContext.ts";
import * as McpSessionRegistry from "./McpSessionRegistry.ts";
import * as PreviewAutomationBroker from "./PreviewAutomationBroker.ts";
@@ -174,12 +175,14 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot
readonly [key: string]: unknown;
};
const { screenshot, ...page } = snapshot;
+ const redacted = redactPreviewSnapshot(page, screenshot);
const metadata = {
- ...page,
+ ...redacted.page,
screenshot: {
mimeType: screenshot.mimeType,
width: screenshot.width,
height: screenshot.height,
+ redacted: redacted.frameRedacted,
},
};
return Effect.succeed(
@@ -190,7 +193,7 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot
{ type: "text", text: JSON.stringify(metadata) },
{
type: "image",
- data: new Uint8Array(Buffer.from(screenshot.data, "base64")),
+ data: redacted.screenshot,
mimeType: screenshot.mimeType,
},
],
diff --git a/apps/server/src/mcp/PreviewSnapshotRedaction.test.ts b/apps/server/src/mcp/PreviewSnapshotRedaction.test.ts
new file mode 100644
index 000000000..186b93b78
--- /dev/null
+++ b/apps/server/src/mcp/PreviewSnapshotRedaction.test.ts
@@ -0,0 +1,114 @@
+import { PNG } from "pngjs";
+import { describe, expect, it, vi } from "vite-plus/test";
+
+import { redactComputerScreenshot, redactPreviewSnapshot } from "./PreviewSnapshotRedaction.ts";
+import { redactSensitiveText } from "./SensitiveDataRedaction.ts";
+
+function testScreenshot() {
+ const png = new PNG({ width: 2, height: 2 });
+ png.data.fill(255);
+ return {
+ mimeType: "image/png" as const,
+ data: PNG.sync.write(png).toString("base64"),
+ width: 2,
+ height: 2,
+ };
+}
+
+function pngChunk(type: string, data: Buffer) {
+ const body = Buffer.concat([Buffer.from(type, "ascii"), data]);
+ let crc = 0xffffffff;
+ for (const byte of body) {
+ crc ^= byte;
+ for (let bit = 0; bit < 8; bit += 1) crc = (crc >>> 1) ^ (0xedb88320 & -(crc & 1));
+ }
+ const chunk = Buffer.alloc(12 + data.length);
+ chunk.writeUInt32BE(data.length, 0);
+ body.copy(chunk, 4);
+ chunk.writeUInt32BE((crc ^ 0xffffffff) >>> 0, 8 + data.length);
+ return chunk;
+}
+
+describe("preview snapshot redaction", () => {
+ it("redacts headers, tokens, emails, phone numbers, and home paths", () => {
+ const result = redactSensitiveText(
+ [
+ "Authorization: Basic dXNlcjpwYXNz",
+ "Cookie: a=one; b=two",
+ "token=secret-value",
+ "/Users/leo/.ssh/id_rsa",
+ "+442071234567",
+ ].join("\n"),
+ );
+
+ expect(result.redacted).toBe(true);
+ expect(result.value).not.toContain("[email protected]");
+ expect(result.value).not.toContain("dXNlcjpwYXNz");
+ expect(result.value).not.toContain("secret-value");
+ expect(result.value).not.toContain("Users/leo");
+ expect(result.value).not.toContain("442071234567");
+ });
+
+ it("redacts secret fields and blanks the screenshot", () => {
+ const result = redactPreviewSnapshot(
+ {
+ cookie: ["opaque"],
+ password: 1234,
+ visible: "safe",
+ consoleOutput: '{"token":"secret-value"}',
+ },
+ testScreenshot(),
+ );
+
+ expect(result.page).toEqual({
+ cookie: "[REDACTED]",
+ password: "[REDACTED]",
+ visible: "safe",
+ consoleOutput: '{"[REDACTED]}',
+ });
+ const png = PNG.sync.read(Buffer.from(result.screenshot));
+ expect([...png.data.subarray(0, 4)]).toEqual([0, 0, 0, 255]);
+ });
+
+ it("rejects oversized dimensions before decoding pixels", () => {
+ const screenshot = testScreenshot();
+ const bytes = Buffer.from(screenshot.data, "base64");
+ bytes.writeUInt32BE(20_000, 16);
+ bytes.writeUInt32BE(20_000, 20);
+ const decode = vi.spyOn(PNG.sync, "read");
+
+ expect(() => redactComputerScreenshot({ mediaType: "image/png", data: bytes })).toThrow(
+ "Screenshot dimensions are invalid.",
+ );
+ expect(decode).not.toHaveBeenCalled();
+ });
+
+ it("fails closed on malformed or mismatched screenshots", () => {
+ const screenshot = testScreenshot();
+ expect(() =>
+ redactComputerScreenshot({
+ mediaType: "image/png",
+ data: Buffer.from("not-a-png"),
+ }),
+ ).toThrow();
+ expect(() => redactPreviewSnapshot({ visible: "safe" }, { ...screenshot, width: 3 })).toThrow(
+ "Preview screenshot dimensions are invalid.",
+ );
+ });
+
+ it("strips PNG ancillary metadata while re-encoding", () => {
+ const screenshot = testScreenshot();
+ const text = Buffer.from("secret metadata", "utf8");
+ const original = Buffer.from(screenshot.data, "base64");
+ const iend = original.length - 12;
+ const bytes = Buffer.concat([
+ original.subarray(0, iend),
+ pngChunk("tEXt", Buffer.concat([Buffer.from("Comment\0"), text])),
+ original.subarray(iend),
+ ]);
+
+ const result = redactComputerScreenshot({ mediaType: "image/png", data: bytes });
+ expect(Buffer.from(result.data).includes(text)).toBe(false);
+ });
+});
diff --git a/apps/server/src/mcp/PreviewSnapshotRedaction.ts b/apps/server/src/mcp/PreviewSnapshotRedaction.ts
new file mode 100644
index 000000000..31e3cf107
--- /dev/null
+++ b/apps/server/src/mcp/PreviewSnapshotRedaction.ts
@@ -0,0 +1,104 @@
+import { PNG } from "pngjs";
+
+import { redactSensitiveText } from "./SensitiveDataRedaction.ts";
+
+const REDACTED = "[REDACTED]";
+const MAX_SCREENSHOT_BYTES = 20 * 1_024 * 1_024;
+const MAX_SCREENSHOT_PIXELS = 16_000_000;
+const secretField =
+ /^(?:api[_-]?key|access[_-]?token|refresh[_-]?token|token|secret|password|authorization|cookie|set-cookie|session|sessionId|clientSecret|awsSecretAccessKey)$/i;
+
+function redactValue(value: unknown, fieldName?: string): { value: unknown; redacted: boolean } {
+ if (fieldName && secretField.test(fieldName)) return { value: REDACTED, redacted: true };
+ if (typeof value === "string") return redactSensitiveText(value);
+ if (Array.isArray(value)) {
+ let redacted = false;
+ const items = value.map((item) => {
+ const result = redactValue(item);
+ redacted ||= result.redacted;
+ return result.value;
+ });
+ return { value: items, redacted };
+ }
+ if (typeof value !== "object" || value === null) return { value, redacted: false };
+
+ let redacted = false;
+ const object: Record<string, unknown> = {};
+ for (const [key, item] of Object.entries(value)) {
+ const result = redactValue(item, key);
+ redacted ||= result.redacted;
+ object[key] = result.value;
+ }
+ return { value: object, redacted };
+}
+
+function readPngDimensions(bytes: Uint8Array) {
+ const buffer = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+ if (buffer.length === 0 || buffer.length > MAX_SCREENSHOT_BYTES) {
+ throw new Error("Screenshot size is invalid.");
+ }
+ if (
+ buffer.length < 24 ||
+ !buffer.subarray(0, 8).equals(Buffer.from([137, 80, 78, 71, 13, 10, 26, 10])) ||
+ buffer.readUInt32BE(8) !== 13 ||
+ buffer.toString("ascii", 12, 16) !== "IHDR"
+ ) {
+ throw new Error("Screenshot is not a valid PNG.");
+ }
+ const width = buffer.readUInt32BE(16);
+ const height = buffer.readUInt32BE(20);
+ if (width === 0 || height === 0 || width * height > MAX_SCREENSHOT_PIXELS) {
+ throw new Error("Screenshot dimensions are invalid.");
+ }
+ return { width, height };
+}
+
+function decodePng(data: Uint8Array) {
+ readPngDimensions(data);
+ return PNG.sync.read(Buffer.from(data), { checkCRC: true });
+}
+
+function blankPng(data: Uint8Array) {
+ const png = decodePng(data);
+ for (let offset = 0; offset < png.data.length; offset += 4) {
+ png.data[offset] = 0;
+ png.data[offset + 1] = 0;
+ png.data[offset + 2] = 0;
+ png.data[offset + 3] = 255;
+ }
+ return PNG.sync.write(png, { colorType: 6, inputColorType: 6, inputHasAlpha: true });
+}
+
+export interface PreviewScreenshotInput {
+ readonly mimeType: "image/png";
+ readonly data: string;
+ readonly width: number;
+ readonly height: number;
+}
+
+export function redactComputerScreenshot(input: {
+ readonly mediaType: "image/png";
+ readonly data: Uint8Array;
+}) {
+ return { mediaType: input.mediaType, data: blankPng(input.data) };
+}
+
+export function redactPreviewSnapshot(
+ page: Readonly<Record<string, unknown>>,
+ screenshot: PreviewScreenshotInput,
+) {
+ const bytes = Buffer.from(screenshot.data, "base64");
+ const dimensions = readPngDimensions(bytes);
+ if (dimensions.width !== screenshot.width || dimensions.height !== screenshot.height) {
+ throw new Error("Preview screenshot dimensions are invalid.");
+ }
+ const redactedPage = redactValue(page);
+ if (typeof redactedPage.value !== "object" || redactedPage.value === null) {
+ throw new Error("Preview snapshot data is invalid.");
+ }
+ return {
+ page: redactedPage.value as Readonly<Record<string, unknown>>,
+ screenshot: blankPng(bytes),
+ frameRedacted: true,
+ };
+}
diff --git a/apps/server/src/mcp/SensitiveDataRedaction.ts b/apps/server/src/mcp/SensitiveDataRedaction.ts
new file mode 100644
index 000000000..a6cb6ac31
--- /dev/null
+++ b/apps/server/src/mcp/SensitiveDataRedaction.ts
@@ -0,0 +1,32 @@
+const REDACTED = "[REDACTED]";
+
+const sensitivePatterns = [
+ /\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/gi,
+ /\b(?:Authorization|Cookie|Set-Cookie)\s*:\s*[^\r\n]+/gi,
+ /\bBearer\s+[A-Z0-9._~+/-]+=*/gi,
+ /\b(?:sk|pk|ghp|github_pat|xox[baprs]|AKIA|glpat|npm)[-_A-Z0-9]{8,}\b/gi,
+ /\beyJ[A-Z0-9_-]+\.[A-Z0-9_-]+\.[A-Z0-9_-]+\b/gi,
+ /\b(?:api[_-]?key|access[_-]?token|refresh[_-]?token|token|secret|password|session(?:Id)?|clientSecret|aws[_-]?secret[_-]?access[_-]?key)\b["']?\s*[:=]\s*[^\r\n,}]+/gi,
+ /\/Users\/[^/\s"'<>]+(?:\/[^\s"'<>]*)?/g,
+ /\/home\/[^/\s"'<>]+(?:\/[^\s"'<>]*)?/g,
+ /\/root(?:\/[^\s"'<>]*)?/g,
+ /~\/(?:[^\s"'<>]*)?/g,
+ /\b[A-Z]:\\Users\\[^\\\s"'<>]+(?:\\[^\s"'<>]*)?/gi,
+ /%2F(?:Users|home)%2F[^\s"'<>]+/gi,
+ /(?<!\d)(?:\+?1[-.\s]?)?(?:\(\d{3}\)|\d{3})[-.\s]\d{3}[-.\s]\d{4}\b/g,
+ /(?<![A-Z0-9])\d{10}(?![A-Z0-9])/gi,
+ /(?<![A-Z0-9])\+\d{10,15}(?![A-Z0-9])/gi,
+ /\+\d{1,3}(?:[-.\s]\d{2,4}){3,5}\b/g,
+] as const;
+
+export function redactSensitiveText(value: string) {
+ let redacted = false;
+ let next = value;
+ for (const pattern of sensitivePatterns) {
+ next = next.replaceAll(pattern, () => {
+ redacted = true;
+ return REDACTED;
+ });
+ }
+ return { value: next, redacted };
+}
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 7e43f3d63..91044b76f 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -508,6 +508,9 @@ importers:
node-pty:
specifier: ^1.1.0
version: 1.1.0
+ pngjs:
+ specifier: 7.0.0
+ version: 7.0.0
yaml:
specifier: ^2.9.0
version: 2.9.0
@@ -533,6 +536,9 @@ importers:
'@types/node':
specifier: 24.12.4
version: 24.12.4
+ '@types/pngjs':
+ specifier: 6.0.5
+ version: 6.0.5
effect-acp:
specifier: workspace:*
version: link:../../packages/effect-acp