Open-source Grok Bot alternative for named AI teammates. Local-first desktop, web, and mobile clients with per-bot tools, instructions, and memory. Independent fork of T3 Code.
import type { PreviewAutomationResponse } from "@akeru/contracts";
import { decodeJson } from "../json.ts";
import * as Predicate from "effect/Predicate";
import {
PreviewAutomationRecordingArtifact,
PreviewAutomationSnapshot,
type PreviewAutomationOperation,
} from "@akeru/contracts";
import { redactSensitiveText } from "@akeru/shared/sensitiveDataRedaction";
import * as Schema from "effect/Schema";
import { PNG } from "pngjs";
const REDACTED = "[REDACTED]";
export const MAX_SCREENSHOT_BYTES = 20 * 1_024 * 1_024;
const MAX_SCREENSHOT_PIXELS = 16_000_000;
const screenshotField = /^(?:screenshot|image|frame)$/i;
const secretField =
/^(?:api[_-]?key|access[_-]?token|refresh[_-]?token|token|secret|password|authorization|cookie|set-cookie|session|sessionId|clientSecret|awsSecretAccessKey|(?:artifact|chat|file|log|recording|upload)?path)$/i;
const decodeSnapshot = Schema.decodeUnknownSync(PreviewAutomationSnapshot);
const decodeRecordingArtifact = Schema.decodeUnknownSync(PreviewAutomationRecordingArtifact);
type PreviewValue =
| string
| number
| boolean
| null
| undefined
| ReadonlyArray<PreviewValue>
| { readonly [key: string]: PreviewValue };
const isPreviewObject = (value: PreviewValue): value is { readonly [key: string]: PreviewValue } =>
Predicate.isObject(value);
function redactValue(value: PreviewValue, fieldName?: string): RedactValueResult {
if (fieldName && secretField.test(fieldName)) return { value: REDACTED, redacted: true };
if (Predicate.isString(value)) return redactSensitiveText(value);
if (Array.isArray(value)) {
let redacted = false;
const items = value.map((item) => {
const result = redactValue(item);
redacted ||= result.redacted;
return result.value;
});
return { value: items, redacted };
}
if (!isPreviewObject(value)) return { value, redacted: false };
let redacted = false;
const entries: Array<[string, PreviewValue]> = [];
for (const [key, item] of Object.entries(value)) {
const result = redactValue(item, key);
redacted ||= result.redacted;
entries.push([key, result.value]);
}
return { value: Object.fromEntries(entries), redacted };
}
function rejectScreenshotPayload(value: PreviewValue, fieldName?: string): void {
if (Predicate.isString(value) && fieldName && screenshotField.test(fieldName)) {
throw new Error("Unredacted screenshot data is not provider-safe.");
}
if (Array.isArray(value)) {
for (const item of value) rejectScreenshotPayload(item, fieldName);
return;
}
if (!isPreviewObject(value)) return;
if (
Object.hasOwn(value, "data") &&
((fieldName !== undefined && screenshotField.test(fieldName)) ||
Object.hasOwn(value, "mimeType"))
) {
throw new Error("Unredacted screenshot data is not provider-safe.");
}
for (const [key, item] of Object.entries(value)) rejectScreenshotPayload(item, key);
}
function readPngDimensions(bytes: Uint8Array) {
const buffer = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);
if (buffer.length === 0 || buffer.length > MAX_SCREENSHOT_BYTES) {
throw new Error("Screenshot size is invalid.");
}
if (
buffer.length < 24 ||
!buffer.subarray(0, 8).equals(Buffer.from([137, 80, 78, 71, 13, 10, 26, 10])) ||
buffer.readUInt32BE(8) !== 13 ||
buffer.toString("ascii", 12, 16) !== "IHDR"
) {
throw new Error("Screenshot is not a valid PNG.");
}
const width = buffer.readUInt32BE(16);
const height = buffer.readUInt32BE(20);
if (width === 0 || height === 0 || width * height > MAX_SCREENSHOT_PIXELS) {
throw new Error("Screenshot dimensions are invalid.");
}
return { width, height };
}
function decodePng(data: Uint8Array) {
readPngDimensions(data);
return PNG.sync.read(Buffer.from(data), { checkCRC: true });
}
function blankPng(data: Uint8Array) {
const png = decodePng(data);
for (let offset = 0; offset < png.data.length; offset += 4) {
png.data[offset] = 0;
png.data[offset + 1] = 0;
png.data[offset + 2] = 0;
png.data[offset + 3] = 255;
}
return PNG.sync.write(png, { colorType: 6, inputColorType: 6, inputHasAlpha: true });
}
export interface PreviewScreenshotInput {
readonly mimeType: "image/png";
readonly data: string;
readonly width: number;
readonly height: number;
}
export function redactComputerScreenshot(input: {
readonly mediaType: "image/png";
readonly data: Uint8Array;
}) {
return { mediaType: input.mediaType, data: blankPng(input.data) };
}
export function redactPreviewSnapshot(
page: Readonly<Record<string, PreviewValue>>,
screenshot: PreviewScreenshotInput,
) {
const bytes = Buffer.from(screenshot.data, "base64");
const dimensions = readPngDimensions(bytes);
if (dimensions.width !== screenshot.width || dimensions.height !== screenshot.height) {
throw new Error("Preview screenshot dimensions are invalid.");
}
const redactedPage = redactValue(page);
if (!isPreviewObject(redactedPage.value)) {
throw new Error("Preview snapshot data is invalid.");
}
return {
page: redactedPage.value,
screenshot: redactComputerScreenshot({ mediaType: "image/png", data: bytes }).data,
frameRedacted: true,
};
}
export function redactProviderVisiblePreviewResult(
operation: PreviewAutomationOperation,
input: PreviewAutomationResponse["result"],
) {
if (operation === "evaluate") {
return { redactionStatus: "omitted-unverified-preview-evaluation" };
}
if (operation === "snapshot") {
const snapshot = decodeSnapshot(input);
const { accessibilityTree: _accessibilityTree, screenshot, ...page } = snapshot;
const redacted = redactPreviewSnapshot(page, screenshot);
return {
...redacted.page,
accessibilityTree: { redactionStatus: "omitted-unverified-accessibility-tree" },
screenshot: {
...screenshot,
data: Buffer.from(redacted.screenshot).toString("base64"),
},
};
}
const value = input === undefined ? undefined : decodeJson(input);
rejectScreenshotPayload(value);
if (operation === "recordingStop") {
return { ...decodeRecordingArtifact(input), path: REDACTED };
}
return redactValue(value).value;
}
type RedactValueResult = { value: PreviewValue; redacted: boolean };